def migration(population, total, k, min_density, csx, csy, **kwargs): """ 'relocation of a portion of a population from one region to another (migration dispersal)' :param da.Array population: population to redistribute :param da.Array k: Carrying Capacity - target location weight and distribution :param float min_density: Each element may have a minimum possible density when relocating population kwargs: source_weight da.Array: Proportion of population to remove from an element. Bounded by `0` and `1` target_weight da.Array: Multiplier for carrying capacity at destination :return: Redistributed population """ if min_density is None: min_density = 0 source_weight = da.clip( kwargs.get("source_weight", da.ones_like(population)), 0, 1) target_weight = da.clip( kwargs.get("target_weight", da.ones_like(population)), 0, 1) # Remove using the source weight removed = population * source_weight new_population = population - removed # Weighting is a combination of k and the target weights target_locations = k * target_weight def redistribute(pop, target, default): # Redistribute population proportionally at the target locations redistribution = target / target.sum() # If there are no target locations, all redistribution values will be inf. # In this case, use the default return da.where( da.isinf(redistribution) | da.isnan(redistribution), default, pop * redistribution) new_population += redistribute(removed.sum(), target_locations, removed) # Remove locations where density is not met density_not_met = (target_locations > 0) & (new_population / target_locations < min_density) removed = da.where(density_not_met, new_population, 0) new_population -= removed new_population += redistribute( removed.sum(), da.where(density_not_met, 0, target_locations), removed) return new_population
def test_clip(): # clip internally calls dd.Series.clip s = pd.Series(np.random.randint(1, 100, size=20)) ds = dd.from_pandas(s, 3) # applying Dask ufunc doesn't trigger computation assert isinstance(da.clip(ds, 5, 50), dd.Series) assert_eq(da.clip(ds, 5, 50), np.clip(s, 5, 50)) # applying Dask ufunc doesn't trigger computation assert isinstance(np.clip(ds, 5, 50), dd.Series) assert_eq(np.clip(ds, 5, 50), np.clip(s, 5, 50)) # applying Dask ufunc to normal Series triggers computation assert isinstance(da.clip(s, 5, 50), pd.Series) assert_eq(da.clip(s, 5, 50), np.clip(s, 5, 50)) df = pd.DataFrame(np.random.randint(1, 100, size=(20, 2)), columns=['A', 'B']) ddf = dd.from_pandas(df, 3) # applying Dask ufunc doesn't trigger computation assert isinstance(da.clip(ddf, 5.5, 40.5), dd.DataFrame) assert_eq(da.clip(ddf, 5.5, 40.5), np.clip(df, 5.5, 40.5)) # applying Dask ufunc doesn't trigger computation assert isinstance(np.clip(ddf, 5.5, 40.5), dd.DataFrame) assert_eq(np.clip(ddf, 5.5, 40.5), np.clip(df, 5.5, 40.5)) # applying Dask ufunc to normal DataFrame triggers computation assert isinstance(da.clip(df, 5.5, 40.5), pd.DataFrame) assert_eq(da.clip(df, 5.5, 40.5), np.clip(df, 5.5, 40.5))
def test_clip(): # clip internally calls dd.Series.clip s = pd.Series(np.random.randint(1, 100, size=20)) ds = dd.from_pandas(s, 3) # applying Dask ufunc doesn't trigger computation assert isinstance(da.clip(ds, 5, 50), dd.Series) assert_eq(da.clip(ds, 5, 50), np.clip(s, 5, 50)) # applying Dask ufunc doesn't trigger computation assert isinstance(np.clip(ds, 5, 50), dd.Series) assert_eq(np.clip(ds, 5, 50), np.clip(s, 5, 50)) # applying Dask ufunc to normal Series triggers computation assert isinstance(da.clip(s, 5, 50), pd.Series) assert_eq(da.clip(s, 5, 50), np.clip(s, 5, 50)) df = pd.DataFrame(np.random.randint(1, 100, size=(20, 2)), columns=['A', 'B']) ddf = dd.from_pandas(df, 3) # applying Dask ufunc doesn't trigger computation assert isinstance(da.clip(ddf, 5.5, 40.5), dd.DataFrame) assert_eq(da.clip(ddf, 5.5, 40.5), np.clip(df, 5.5, 40.5)) # applying Dask ufunc doesn't trigger computation assert isinstance(np.clip(ddf, 5.5, 40.5), dd.DataFrame) assert_eq(np.clip(ddf, 5.5, 40.5), np.clip(df, 5.5, 40.5)) # applying Dask ufunc to normal DataFrame triggers computation assert isinstance(da.clip(df, 5.5, 40.5), pd.DataFrame) assert_eq(da.clip(df, 5.5, 40.5), np.clip(df, 5.5, 40.5))
def discretize(x_data, n_state, min_=0.0, max_=1.0, chunks=()): """Primitive discretization of a microstructure. Args: x_data: the data to discrtize n_state: the number of local states min_: the minimum local state max_: the maximum local state Returns: the discretized microstructure >>> discretize(da.random.random((12, 9), chunks=(3, 9)), ... 3, ... chunks=(1,)).chunks ((3, 3, 3, 3), (9,), (1, 1, 1)) >>> discretize(np.array([[0, 1], [0.5, 0.5]]), 3, chunks=(1,)).chunks ((2,), (2,), (1, 1, 1)) >>> discretize(np.array([[0, 1], [0.5, 0.5]]), 3, chunks=(1,)).compute() array([[[ 1., 0., 0.], [ 0., 0., 1.]], <BLANKLINE> [[ 0., 1., 0.], [ 0., 1., 0.]]]) """ return da.maximum( discretize_nomax( da.clip(x_data, min_, max_), da.linspace(min_, max_, n_state, chunks=chunks or (n_state, ))), 0)
def rescale_amplitude_range(self, darray, min_val, max_val, preview=None): """ Description ----------- Clip the seismic data to specified values Parameters ---------- darray : Array-like, acceptable inputs include Numpy, HDF5, or Dask Arrays min_val : Number, min clip value max_val : Numer, max clip value Keywork Arguments ----------------- preview : str, enables or disables preview mode and specifies direction Acceptable inputs are (None, 'inline', 'xline', 'z') Optimizes chunk size in different orientations to facilitate rapid screening of algorithm output Returns ------- result : Dask Array """ darray, chunks_init = self.create_array(darray, preview=preview) result = da.clip(darray, min_val, max_val) return (result)
def relative_amplitude_change(self, darray, preview=None): """ Description ----------- Compute the Relative Amplitude Change of the input data Parameters ---------- darray : Array-like, acceptable inputs include Numpy, HDF5, or Dask Arrays Keywork Arguments ----------------- preview : str, enables or disables preview mode and specifies direction Acceptable inputs are (None, 'inline', 'xline', 'z') Optimizes chunk size in different orientations to facilitate rapid screening of algorithm output Returns ------- result : Dask Array """ darray, chunks_init = self.create_array(darray, preview=preview) env = self.envelope(darray) env_prime = sp().first_derivative(env, axis=-1) result = env_prime / env result = da.clip(result, -1, 1) return (result)
def concat_bands(self): import dask.array as da self.logger.info('Concatenate RGB bands...') self._arrays['rgb'] = da.dstack((self._arrays['r'], self._arrays['g'], self._arrays['b'])) self._arrays['rgb'] = da.clip(self._arrays['rgb'], 0, 1) del self._arrays['r'], self._arrays['g'], self._arrays['b']
def clip(cube, minimum=None, maximum=None): """ Clip values at a specified minimum and/or maximum value Values lower than minimum are set to minimum and values higher than maximum are set to maximum. Parameters ---------- cube: iris.cube.Cube iris cube to be clipped minimum: float lower threshold to be applied on input cube data. maximum: float upper threshold to be applied on input cube data. Returns ------- iris.cube.Cube clipped cube. """ if minimum is None and maximum is None: raise ValueError("Either minimum, maximum or both have to be\ specified.") elif minimum is not None and maximum is not None: if maximum < minimum: raise ValueError("Maximum should be equal or larger than minimum.") cube.data = da.clip(cube.core_data(), minimum, maximum) return cube
def slerp(self, val, low, high): """Code from https://github.com/soumith/dcgan.torch/issues/14""" omega = da.arccos(da.clip(da.dot(low / da.linalg.norm(low), high.transpose() / da.linalg.norm(high)), -1, 1)) so = da.sin(omega) if so == 0: return (1.0-val) * low + val * high # L'Hopital's rule/LERP return da.sin((1.0 - val) * omega) / so * low + da.sin(val * omega) / so * high
def func(band_data, index=None): idx = np.array(kwargs['idx']) / 255 sc = np.array(kwargs['sc']) / 255 band_data *= .01 # Interpolate band on [0,1] using "lazy" arrays (put calculations off until the end). band_data = xr.DataArray(da.clip(band_data.data.map_blocks(np.interp, xp=idx, fp=sc), 0, 1), coords=band_data.coords, dims=band_data.dims, name=band_data.name, attrs=band_data.attrs) return band_data
def func(band_data, xp, fp, index=None): # Interpolate band on [0,1] using "lazy" arrays (put calculations off until the end). band_data = xr.DataArray(da.clip( band_data.data.map_blocks(np.interp, xp=xp, fp=fp), 0, 1), coords=band_data.coords, dims=band_data.dims, name=band_data.name, attrs=band_data.attrs) return band_data
def test_clip(pandas, min, max): dask = dd.from_pandas(pandas, 3) pandas_type = pandas.__class__ dask_type = dask.__class__ # clip internally calls dd.Series.clip # applying Dask ufunc doesn't trigger computation assert isinstance(da.clip(dask, min, max), dask_type) assert_eq(da.clip(dask, min, max), np.clip(pandas, min, max)) # applying Numpy ufunc doesn't trigger computation assert isinstance(np.clip(dask, min, max), dask_type) assert_eq(np.clip(dask, min, max), np.clip(pandas, min, max)) # applying Dask ufunc to normal pandas objects triggers computation assert isinstance(da.clip(pandas, min, max), pandas_type) assert_eq(da.clip(pandas, min, max), np.clip(pandas, min, max))
def test_clip(pandas, min, max): dask = dd.from_pandas(pandas, 3) pandas_type = pandas.__class__ dask_type = dask.__class__ # clip internally calls dd.Series.clip # applying Dask ufunc doesn't trigger computation assert isinstance(da.clip(dask, min, max), dask_type) assert_eq(da.clip(dask, min, max), np.clip(pandas, min, max)) # applying Numpy ufunc doesn't trigger computation assert isinstance(np.clip(dask, min, max), dask_type) assert_eq(np.clip(dask, min, max), np.clip(pandas, min, max)) # applying Dask ufunc to normal pandas objects triggers computation assert isinstance(da.clip(pandas, min, max), pandas_type) assert_eq(da.clip(pandas, min, max), np.clip(pandas, min, max))
def _transform_col(self, X_col: da.Array, quantiles: ArrayLike, inverse: bool) -> ArrayLike: output_distribution = self.output_distribution if not inverse: lower_bound_x = quantiles[0] upper_bound_x = quantiles[-1] lower_bound_y = 0 upper_bound_y = 1 else: lower_bound_x = 0 upper_bound_x = 1 lower_bound_y = quantiles[0] upper_bound_y = quantiles[-1] # for inverse transform, match a uniform distribution if output_distribution == "normal": X_col = X_col.map_blocks(stats.norm.cdf) # else output distribution is already a uniform distribution if output_distribution == "normal": lower_bounds_idx = X_col - BOUNDS_THRESHOLD < lower_bound_x upper_bounds_idx = X_col + BOUNDS_THRESHOLD > upper_bound_x if output_distribution == "uniform": lower_bounds_idx = X_col == lower_bound_x upper_bounds_idx = X_col == upper_bound_x if not inverse: # See the note in scikit-learn. This trick is to avoid # repeated extreme values X_col = 0.5 * ( X_col.map_blocks(np.interp, quantiles, self.references_) - (-X_col).map_blocks(np.interp, -quantiles[::-1], -self.references_[::-1])) else: X_col = X_col.map_blocks(np.interp, self.references_, quantiles) X_col[upper_bounds_idx] = upper_bound_y X_col[lower_bounds_idx] = lower_bound_y if not inverse: if output_distribution == "normal": X_col = X_col.map_blocks(stats.norm.ppf) # find the value to clip the data to avoid mapping to # infinity. Clip such that the inverse transform will be # consistent clip_min = stats.norm.ppf(BOUNDS_THRESHOLD - np.spacing(1)) clip_max = stats.norm.ppf(1 - (BOUNDS_THRESHOLD - np.spacing(1))) X_col = da.clip(X_col, clip_min, clip_max) # else output distribution is uniform and the ppf is the # identity function so we let X_col unchanged return X_col
def get_phaang(cos_vza, cos_sza, sin_vza, sin_sza, cos_raa): """ Gets the phase angle """ cos_phase_angle = da.clip( cos_vza * cos_sza + sin_vza * sin_sza * cos_raa, -1, 1) phase_angle = da.arccos(cos_phase_angle) sin_phase_angle = da.sin(phase_angle) return cos_phase_angle, phase_angle, sin_phase_angle
def _transform_col(self, X_col, quantiles, inverse): if self.output_distribution == 'normal': output_distribution = 'norm' else: output_distribution = self.output_distribution output_distribution = getattr(stats, output_distribution) if not inverse: lower_bound_x = quantiles[0] upper_bound_x = quantiles[-1] lower_bound_y = 0 upper_bound_y = 1 else: lower_bound_x = 0 upper_bound_x = 1 lower_bound_y = quantiles[0] upper_bound_y = quantiles[-1] X_col = X_col.map_blocks(output_distribution.cdf) lower_bounds_idx = (X_col - skdata.BOUNDS_THRESHOLD < lower_bound_x) upper_bounds_idx = (X_col + skdata.BOUNDS_THRESHOLD > upper_bound_x) if not inverse: # See the note in scikit-learn. This trick is to avoid # repeated extreme values X_col = 0.5 * ( X_col.map_blocks(np.interp, quantiles, self.references_) - (-X_col).map_blocks(np.interp, -quantiles[::-1], -self.references_[::-1]) ) else: X_col = X_col.map_blocks(np.interp, self.references_, quantiles) X_col[upper_bounds_idx] = upper_bound_y X_col[lower_bounds_idx] = lower_bound_y if not inverse: X_col = X_col.map_blocks(output_distribution.ppf) clip_min = output_distribution.ppf(skdata.BOUNDS_THRESHOLD - np.spacing(1)) clip_max = output_distribution.ppf(1 - (skdata.BOUNDS_THRESHOLD - np.spacing(1))) X_col = da.clip(X_col, clip_min, clip_max) return X_col
def get_dense_mdm(self) -> np.ndarray: r""" Form the Mapped Distance Matrix as a dense Numpy array. Returns ------- np.ndarray Mapped Distance Matrix as a Numpy array. """ if self.mode == 'radial': distances = da.linalg.norm(self.samples1[:, None, :] - self.samples2[None, ...], axis=-1) elif self.mode == 'zonal': distances = da.clip(da.sum(self.samples1[:, None, :] * self.samples2[None, ...], axis=-1), a_min=-1, a_max=1) else: raise ValueError(f'Unsupported mode {self.mode}.') mapped_distance_matrix = self.function(distances) return mapped_distance_matrix.astype(self.dtype)
def convert_dtype(in_data, min_val, max_val, to_dtype): """ Description ----------- Convience function to read file and create a pointer to data on disk Parameters ---------- in_data : Dask Array, data to convert min_val : number, lower clip max_val : number, upper clip to_dtype : NumPy dtype Acceptable formats include (np.int8, np.float16, np.float32) Returns ------- out : Dask Array, converted data """ # Check if data is already in correct format if in_data.dtype == to_dtype: return (in_data) else: in_data = da.clip(in_data, min_val, max_val) if to_dtype == np.int8: in_data = ((in_data - min_val) / (max_val - min_val)) dtype = np.iinfo(np.int8) in_data *= dtype.max - dtype.min in_data -= dtype.min out = in_data.astype(np.int8) elif to_dtype == np.float16: out = in_data.astype(np.float16) elif to_dtype == np.int32: out = in_data.astype(np.float32) else: raise Exception('Not a valid dtype') return (out)
def single_channel_percentile_norm(data: da.core.Array, min_p: float = 50.0, max_p: float = 99.8, **kwargs) -> da.core.Array: # Enforce shape if len(data.shape) > 4: raise exceptions.InvalidShapeError(len(data.shape), 4) # Get the norm by values norm_by = da.percentile(data.flatten(), [min_p, min_p]).compute() # Norm normed = (data - norm_by[0]) / (norm_by[1] - norm_by[0]) # Clip any values outside of 0 and 1 clipped = da.clip(normed, 0, 1) # Scale them between 0 and 255 return clipped * 255
def get_dask_mdm(self) -> da.core.Array: r""" Form the Mapped Distance Matrix as a dask array. Returns ------- da.core.Array Mapped Distance Matrix as a Dask array. """ samples_da = da.from_array(self.samples1[:, None, :], chunks=self.chunks) knots_da = da.from_array(self.samples2[None, ...], chunks=self.chunks) if self.mode == 'radial': distances_da = da.linalg.norm(samples_da - knots_da, axis=-1) elif self.mode == 'zonal': distances_da = da.clip(da.sum(samples_da * knots_da, axis=-1), a_min=-1, a_max=1) else: raise ValueError(f'Unsupported mode {self.mode}.') mapped_distance_matrix = da.map_blocks(self.function, distances_da, dtype=self.dtype) return mapped_distance_matrix
def get_overlap(cos1, cos2, tan1, tan2, sin3, distance, hb, m_pi): """ Applies the HB ratio transformation """ OverlapInfo = namedtuple('OverlapInfo', 'tvar sint overlap temp') temp = (1.0 / cos1) + (1.0 / cos2) cost = da.clip( hb * da.sqrt(distance * distance + tan1 * tan1 * tan2 * tan2 * sin3 * sin3) / temp, -1, 1) tvar = da.arccos(cost) sint = da.sin(tvar) overlap = 1.0 / m_pi * (tvar - sint * cost) * temp overlap = da.maximum(overlap, 0) return OverlapInfo(tvar=tvar, sint=sint, overlap=overlap, temp=temp)
def discretize(x_data, n_state=2, min_=0.0, max_=1.0, chunks=None): """Primitive discretization of a microstructure. Args: x_data: the data to discretize n_state: the number of local states min_: the minimum local state max_: the maximum local state chunks: chunks size for state axis Returns: the discretized microstructure >>> discretize(da.random.random((12, 9), chunks=(3, 9)), ... 3, ... chunks=1).chunks ((3, 3, 3, 3), (9,), (1, 1, 1)) >>> discretize(np.array([[0, 1], [0.5, 0.5]]), 3, chunks=1).chunks ((2,), (2,), (1, 1, 1)) >>> assert np.allclose( ... discretize( ... np.array([[0, 1], [0.5, 0.5]]), ... 3, ... chunks=1 ... ).compute(), ... [[[1, 0, 0], [0, 0, 1]], [[0, 1, 0], [0, 1, 0]]] ... ) """ return da.maximum( discretize_nomax( da.clip(x_data, min_, max_), da.linspace(min_, max_, n_state, chunks=(chunks or n_state,)), ), 0, )
def apply_geometric_transform(img: da.Array, dark: da.Array, flat: da.Array, cof_dist: Dict[str, List[float]]) -> da.Array: """Apply geometric transformation to array Parameters ---------- img Input image flat flat field image cof_dist Distortion coefficients Returns ------- distortion corrected array """ norm = da.flipud((img - dark) / da.clip(flat, 1e-6, 1e6)) masked = delayed(mask_image)(norm) masked = da.from_delayed(masked, shape=norm.shape, dtype="float32") matrix = get_matrix(masked.shape, cof_dist) new = delayed(warp)(masked, matrix) res = da.from_delayed(new, shape=img.shape, dtype="float32") return res
def test_arithmetic(): x = np.arange(5).astype('f4') + 2 y = np.arange(5).astype('i8') + 2 z = np.arange(5).astype('i4') + 2 a = da.from_array(x, chunks=(2,)) b = da.from_array(y, chunks=(2,)) c = da.from_array(z, chunks=(2,)) assert eq(a + b, x + y) assert eq(a * b, x * y) assert eq(a - b, x - y) assert eq(a / b, x / y) assert eq(b & b, y & y) assert eq(b | b, y | y) assert eq(b ^ b, y ^ y) assert eq(a // b, x // y) assert eq(a ** b, x ** y) assert eq(a % b, x % y) assert eq(a > b, x > y) assert eq(a < b, x < y) assert eq(a >= b, x >= y) assert eq(a <= b, x <= y) assert eq(a == b, x == y) assert eq(a != b, x != y) assert eq(a + 2, x + 2) assert eq(a * 2, x * 2) assert eq(a - 2, x - 2) assert eq(a / 2, x / 2) assert eq(b & True, y & True) assert eq(b | True, y | True) assert eq(b ^ True, y ^ True) assert eq(a // 2, x // 2) assert eq(a ** 2, x ** 2) assert eq(a % 2, x % 2) assert eq(a > 2, x > 2) assert eq(a < 2, x < 2) assert eq(a >= 2, x >= 2) assert eq(a <= 2, x <= 2) assert eq(a == 2, x == 2) assert eq(a != 2, x != 2) assert eq(2 + b, 2 + y) assert eq(2 * b, 2 * y) assert eq(2 - b, 2 - y) assert eq(2 / b, 2 / y) assert eq(True & b, True & y) assert eq(True | b, True | y) assert eq(True ^ b, True ^ y) assert eq(2 // b, 2 // y) assert eq(2 ** b, 2 ** y) assert eq(2 % b, 2 % y) assert eq(2 > b, 2 > y) assert eq(2 < b, 2 < y) assert eq(2 >= b, 2 >= y) assert eq(2 <= b, 2 <= y) assert eq(2 == b, 2 == y) assert eq(2 != b, 2 != y) assert eq(-a, -x) assert eq(abs(a), abs(x)) assert eq(~(a == b), ~(x == y)) assert eq(~(a == b), ~(x == y)) assert eq(da.logaddexp(a, b), np.logaddexp(x, y)) assert eq(da.logaddexp2(a, b), np.logaddexp2(x, y)) assert eq(da.exp(b), np.exp(y)) assert eq(da.log(a), np.log(x)) assert eq(da.log10(a), np.log10(x)) assert eq(da.log1p(a), np.log1p(x)) assert eq(da.expm1(b), np.expm1(y)) assert eq(da.sqrt(a), np.sqrt(x)) assert eq(da.square(a), np.square(x)) assert eq(da.sin(a), np.sin(x)) assert eq(da.cos(b), np.cos(y)) assert eq(da.tan(a), np.tan(x)) assert eq(da.arcsin(b/10), np.arcsin(y/10)) assert eq(da.arccos(b/10), np.arccos(y/10)) assert eq(da.arctan(b/10), np.arctan(y/10)) assert eq(da.arctan2(b*10, a), np.arctan2(y*10, x)) assert eq(da.hypot(b, a), np.hypot(y, x)) assert eq(da.sinh(a), np.sinh(x)) assert eq(da.cosh(b), np.cosh(y)) assert eq(da.tanh(a), np.tanh(x)) assert eq(da.arcsinh(b*10), np.arcsinh(y*10)) assert eq(da.arccosh(b*10), np.arccosh(y*10)) assert eq(da.arctanh(b/10), np.arctanh(y/10)) assert eq(da.deg2rad(a), np.deg2rad(x)) assert eq(da.rad2deg(a), np.rad2deg(x)) assert eq(da.logical_and(a < 1, b < 4), np.logical_and(x < 1, y < 4)) assert eq(da.logical_or(a < 1, b < 4), np.logical_or(x < 1, y < 4)) assert eq(da.logical_xor(a < 1, b < 4), np.logical_xor(x < 1, y < 4)) assert eq(da.logical_not(a < 1), np.logical_not(x < 1)) assert eq(da.maximum(a, 5 - a), np.maximum(a, 5 - a)) assert eq(da.minimum(a, 5 - a), np.minimum(a, 5 - a)) assert eq(da.fmax(a, 5 - a), np.fmax(a, 5 - a)) assert eq(da.fmin(a, 5 - a), np.fmin(a, 5 - a)) assert eq(da.isreal(a + 1j * b), np.isreal(x + 1j * y)) assert eq(da.iscomplex(a + 1j * b), np.iscomplex(x + 1j * y)) assert eq(da.isfinite(a), np.isfinite(x)) assert eq(da.isinf(a), np.isinf(x)) assert eq(da.isnan(a), np.isnan(x)) assert eq(da.signbit(a - 3), np.signbit(x - 3)) assert eq(da.copysign(a - 3, b), np.copysign(x - 3, y)) assert eq(da.nextafter(a - 3, b), np.nextafter(x - 3, y)) assert eq(da.ldexp(c, c), np.ldexp(z, z)) assert eq(da.fmod(a * 12, b), np.fmod(x * 12, y)) assert eq(da.floor(a * 0.5), np.floor(x * 0.5)) assert eq(da.ceil(a), np.ceil(x)) assert eq(da.trunc(a / 2), np.trunc(x / 2)) assert eq(da.degrees(b), np.degrees(y)) assert eq(da.radians(a), np.radians(x)) assert eq(da.rint(a + 0.3), np.rint(x + 0.3)) assert eq(da.fix(a - 2.5), np.fix(x - 2.5)) assert eq(da.angle(a + 1j), np.angle(x + 1j)) assert eq(da.real(a + 1j), np.real(x + 1j)) assert eq((a + 1j).real, np.real(x + 1j)) assert eq(da.imag(a + 1j), np.imag(x + 1j)) assert eq((a + 1j).imag, np.imag(x + 1j)) assert eq(da.conj(a + 1j * b), np.conj(x + 1j * y)) assert eq((a + 1j * b).conj(), (x + 1j * y).conj()) assert eq(da.clip(b, 1, 4), np.clip(y, 1, 4)) assert eq(da.fabs(b), np.fabs(y)) assert eq(da.sign(b - 2), np.sign(y - 2)) l1, l2 = da.frexp(a) r1, r2 = np.frexp(x) assert eq(l1, r1) assert eq(l2, r2) l1, l2 = da.modf(a) r1, r2 = np.modf(x) assert eq(l1, r1) assert eq(l2, r2) assert eq(da.around(a, -1), np.around(x, -1))
def get_reflectance(self, sun_zenith, sat_zenith, azidiff, bandname, redband=None): """Get the reflectance from the three sun-sat angles""" # Get wavelength in nm for band: if isinstance(bandname, float): LOG.warning('A wavelength is provided instead of band name - ' + 'disregard the relative spectral responses and assume ' + 'it is the effective wavelength: %f (micro meter)', bandname) wvl = bandname * 1000.0 else: wvl = self.get_effective_wavelength(bandname) wvl = wvl * 1000.0 rayl, wvl_coord, azid_coord, satz_sec_coord, sunz_sec_coord = self.get_reflectance_lut() # force dask arrays compute = False if HAVE_DASK and not isinstance(sun_zenith, Array): compute = True sun_zenith = from_array(sun_zenith, chunks=sun_zenith.shape) sat_zenith = from_array(sat_zenith, chunks=sat_zenith.shape) azidiff = from_array(azidiff, chunks=azidiff.shape) if redband is not None: redband = from_array(redband, chunks=redband.shape) clip_angle = rad2deg(arccos(1. / sunz_sec_coord.max())) sun_zenith = clip(sun_zenith, 0, clip_angle) sunzsec = 1. / cos(deg2rad(sun_zenith)) clip_angle = rad2deg(arccos(1. / satz_sec_coord.max())) sat_zenith = clip(sat_zenith, 0, clip_angle) satzsec = 1. / cos(deg2rad(sat_zenith)) shape = sun_zenith.shape if not(wvl_coord.min() < wvl < wvl_coord.max()): LOG.warning( "Effective wavelength for band %s outside 400-800 nm range!", str(bandname)) LOG.info( "Set the rayleigh/aerosol reflectance contribution to zero!") if HAVE_DASK: chunks = sun_zenith.chunks if redband is None else redband.chunks res = zeros(shape, chunks=chunks) return res.compute() if compute else res else: return zeros(shape) idx = np.searchsorted(wvl_coord, wvl) wvl1 = wvl_coord[idx - 1] wvl2 = wvl_coord[idx] fac = (wvl2 - wvl) / (wvl2 - wvl1) raylwvl = fac * rayl[idx - 1, :, :, :] + (1 - fac) * rayl[idx, :, :, :] tic = time.time() smin = [sunz_sec_coord[0], azid_coord[0], satz_sec_coord[0]] smax = [sunz_sec_coord[-1], azid_coord[-1], satz_sec_coord[-1]] orders = [ len(sunz_sec_coord), len(azid_coord), len(satz_sec_coord)] f_3d_grid = atleast_2d(raylwvl.ravel()) if HAVE_DASK and isinstance(smin[0], Array): # compute all of these at the same time before passing to the interpolator # otherwise they are computed separately smin, smax, orders, f_3d_grid = da.compute(smin, smax, orders, f_3d_grid) minterp = MultilinearInterpolator(smin, smax, orders) minterp.set_values(f_3d_grid) if HAVE_DASK: ipn = map_blocks(self._do_interp, minterp, sunzsec, azidiff, satzsec, dtype=raylwvl.dtype, chunks=azidiff.chunks) else: ipn = self._do_interp(minterp, sunzsec, azidiff, satzsec) LOG.debug("Time - Interpolation: {0:f}".format(time.time() - tic)) ipn *= 100 res = ipn if redband is not None: res = where(redband < 20., res, (1 - (redband - 20) / 80) * res) res = clip(res, 0, 100) if compute: res = res.compute() return res
def _estimate_poissonian_noise_variance(dc, gain_factor, gain_offset, correlation_factor): variance = (dc * gain_factor + gain_offset) * correlation_factor # The lower bound of the variance is the gaussian noise. variance = da.clip(variance, gain_offset * correlation_factor, np.inf) return variance
def get_reflectance(self, sun_zenith, sat_zenith, azidiff, bandname, redband=None): """Get the reflectance from the three sun-sat angles""" # Get wavelength in nm for band: if isinstance(bandname, float): LOG.warning('A wavelength is provided instead of band name - ' + 'disregard the relative spectral responses and assume ' + 'it is the effective wavelength: %f (micro meter)', bandname) wvl = bandname * 1000.0 else: wvl = self.get_effective_wavelength(bandname) if wvl is None: LOG.error("Can't get effective wavelength for band %s on platform %s and sensor %s", str(bandname), self.platform_name, self.sensor) return None else: wvl = wvl * 1000.0 rayl, wvl_coord, azid_coord, satz_sec_coord, sunz_sec_coord = \ self.get_reflectance_lut() # force dask arrays compute = False if HAVE_DASK and not isinstance(sun_zenith, Array): compute = True sun_zenith = from_array(sun_zenith, chunks=sun_zenith.shape) sat_zenith = from_array(sat_zenith, chunks=sat_zenith.shape) azidiff = from_array(azidiff, chunks=azidiff.shape) if redband is not None: redband = from_array(redband, chunks=redband.shape) clip_angle = rad2deg(arccos(1. / sunz_sec_coord.max())) sun_zenith = clip(sun_zenith, 0, clip_angle) sunzsec = 1. / cos(deg2rad(sun_zenith)) clip_angle = rad2deg(arccos(1. / satz_sec_coord.max())) sat_zenith = clip(sat_zenith, 0, clip_angle) satzsec = 1. / cos(deg2rad(sat_zenith)) shape = sun_zenith.shape if not(wvl_coord.min() < wvl < wvl_coord.max()): LOG.warning( "Effective wavelength for band %s outside 400-800 nm range!", str(bandname)) LOG.info( "Set the rayleigh/aerosol reflectance contribution to zero!") if HAVE_DASK: chunks = sun_zenith.chunks if redband is None \ else redband.chunks res = zeros(shape, chunks=chunks) return res.compute() if compute else res else: return zeros(shape) idx = np.searchsorted(wvl_coord, wvl) wvl1 = wvl_coord[idx - 1] wvl2 = wvl_coord[idx] fac = (wvl2 - wvl) / (wvl2 - wvl1) raylwvl = fac * rayl[idx - 1, :, :, :] + (1 - fac) * rayl[idx, :, :, :] tic = time.time() smin = [sunz_sec_coord[0], azid_coord[0], satz_sec_coord[0]] smax = [sunz_sec_coord[-1], azid_coord[-1], satz_sec_coord[-1]] orders = [ len(sunz_sec_coord), len(azid_coord), len(satz_sec_coord)] f_3d_grid = atleast_2d(raylwvl.ravel()) if HAVE_DASK and isinstance(smin[0], Array): # compute all of these at the same time before passing to the interpolator # otherwise they are computed separately smin, smax, orders, f_3d_grid = da.compute(smin, smax, orders, f_3d_grid) minterp = MultilinearInterpolator(smin, smax, orders) minterp.set_values(f_3d_grid) def _do_interp(minterp, sunzsec, azidiff, satzsec): interp_points2 = np.vstack((sunzsec.ravel(), 180 - azidiff.ravel(), satzsec.ravel())) res = minterp(interp_points2) return res.reshape(sunzsec.shape) if HAVE_DASK: ipn = map_blocks(_do_interp, minterp, sunzsec, azidiff, satzsec, dtype=raylwvl.dtype, chunks=azidiff.chunks) else: ipn = _do_interp(minterp, sunzsec, azidiff, satzsec) LOG.debug("Time - Interpolation: {0:f}".format(time.time() - tic)) ipn *= 100 res = ipn if redband is not None: res = where(redband < 20., res, (1 - (redband - 20) / 80) * res) res = clip(res, 0, 100) if compute: res = res.compute() return res
def test_arithmetic(): x = np.arange(5).astype('f4') + 2 y = np.arange(5).astype('i8') + 2 z = np.arange(5).astype('i4') + 2 a = da.from_array(x, chunks=(2, )) b = da.from_array(y, chunks=(2, )) c = da.from_array(z, chunks=(2, )) assert eq(a + b, x + y) assert eq(a * b, x * y) assert eq(a - b, x - y) assert eq(a / b, x / y) assert eq(b & b, y & y) assert eq(b | b, y | y) assert eq(b ^ b, y ^ y) assert eq(a // b, x // y) assert eq(a**b, x**y) assert eq(a % b, x % y) assert eq(a > b, x > y) assert eq(a < b, x < y) assert eq(a >= b, x >= y) assert eq(a <= b, x <= y) assert eq(a == b, x == y) assert eq(a != b, x != y) assert eq(a + 2, x + 2) assert eq(a * 2, x * 2) assert eq(a - 2, x - 2) assert eq(a / 2, x / 2) assert eq(b & True, y & True) assert eq(b | True, y | True) assert eq(b ^ True, y ^ True) assert eq(a // 2, x // 2) assert eq(a**2, x**2) assert eq(a % 2, x % 2) assert eq(a > 2, x > 2) assert eq(a < 2, x < 2) assert eq(a >= 2, x >= 2) assert eq(a <= 2, x <= 2) assert eq(a == 2, x == 2) assert eq(a != 2, x != 2) assert eq(2 + b, 2 + y) assert eq(2 * b, 2 * y) assert eq(2 - b, 2 - y) assert eq(2 / b, 2 / y) assert eq(True & b, True & y) assert eq(True | b, True | y) assert eq(True ^ b, True ^ y) assert eq(2 // b, 2 // y) assert eq(2**b, 2**y) assert eq(2 % b, 2 % y) assert eq(2 > b, 2 > y) assert eq(2 < b, 2 < y) assert eq(2 >= b, 2 >= y) assert eq(2 <= b, 2 <= y) assert eq(2 == b, 2 == y) assert eq(2 != b, 2 != y) assert eq(-a, -x) assert eq(abs(a), abs(x)) assert eq(~(a == b), ~(x == y)) assert eq(~(a == b), ~(x == y)) assert eq(da.logaddexp(a, b), np.logaddexp(x, y)) assert eq(da.logaddexp2(a, b), np.logaddexp2(x, y)) assert eq(da.exp(b), np.exp(y)) assert eq(da.log(a), np.log(x)) assert eq(da.log10(a), np.log10(x)) assert eq(da.log1p(a), np.log1p(x)) assert eq(da.expm1(b), np.expm1(y)) assert eq(da.sqrt(a), np.sqrt(x)) assert eq(da.square(a), np.square(x)) assert eq(da.sin(a), np.sin(x)) assert eq(da.cos(b), np.cos(y)) assert eq(da.tan(a), np.tan(x)) assert eq(da.arcsin(b / 10), np.arcsin(y / 10)) assert eq(da.arccos(b / 10), np.arccos(y / 10)) assert eq(da.arctan(b / 10), np.arctan(y / 10)) assert eq(da.arctan2(b * 10, a), np.arctan2(y * 10, x)) assert eq(da.hypot(b, a), np.hypot(y, x)) assert eq(da.sinh(a), np.sinh(x)) assert eq(da.cosh(b), np.cosh(y)) assert eq(da.tanh(a), np.tanh(x)) assert eq(da.arcsinh(b * 10), np.arcsinh(y * 10)) assert eq(da.arccosh(b * 10), np.arccosh(y * 10)) assert eq(da.arctanh(b / 10), np.arctanh(y / 10)) assert eq(da.deg2rad(a), np.deg2rad(x)) assert eq(da.rad2deg(a), np.rad2deg(x)) assert eq(da.logical_and(a < 1, b < 4), np.logical_and(x < 1, y < 4)) assert eq(da.logical_or(a < 1, b < 4), np.logical_or(x < 1, y < 4)) assert eq(da.logical_xor(a < 1, b < 4), np.logical_xor(x < 1, y < 4)) assert eq(da.logical_not(a < 1), np.logical_not(x < 1)) assert eq(da.maximum(a, 5 - a), np.maximum(a, 5 - a)) assert eq(da.minimum(a, 5 - a), np.minimum(a, 5 - a)) assert eq(da.fmax(a, 5 - a), np.fmax(a, 5 - a)) assert eq(da.fmin(a, 5 - a), np.fmin(a, 5 - a)) assert eq(da.isreal(a + 1j * b), np.isreal(x + 1j * y)) assert eq(da.iscomplex(a + 1j * b), np.iscomplex(x + 1j * y)) assert eq(da.isfinite(a), np.isfinite(x)) assert eq(da.isinf(a), np.isinf(x)) assert eq(da.isnan(a), np.isnan(x)) assert eq(da.signbit(a - 3), np.signbit(x - 3)) assert eq(da.copysign(a - 3, b), np.copysign(x - 3, y)) assert eq(da.nextafter(a - 3, b), np.nextafter(x - 3, y)) assert eq(da.ldexp(c, c), np.ldexp(z, z)) assert eq(da.fmod(a * 12, b), np.fmod(x * 12, y)) assert eq(da.floor(a * 0.5), np.floor(x * 0.5)) assert eq(da.ceil(a), np.ceil(x)) assert eq(da.trunc(a / 2), np.trunc(x / 2)) assert eq(da.degrees(b), np.degrees(y)) assert eq(da.radians(a), np.radians(x)) assert eq(da.rint(a + 0.3), np.rint(x + 0.3)) assert eq(da.fix(a - 2.5), np.fix(x - 2.5)) assert eq(da.angle(a + 1j), np.angle(x + 1j)) assert eq(da.real(a + 1j), np.real(x + 1j)) assert eq((a + 1j).real, np.real(x + 1j)) assert eq(da.imag(a + 1j), np.imag(x + 1j)) assert eq((a + 1j).imag, np.imag(x + 1j)) assert eq(da.conj(a + 1j * b), np.conj(x + 1j * y)) assert eq((a + 1j * b).conj(), (x + 1j * y).conj()) assert eq(da.clip(b, 1, 4), np.clip(y, 1, 4)) assert eq(da.fabs(b), np.fabs(y)) assert eq(da.sign(b - 2), np.sign(y - 2)) l1, l2 = da.frexp(a) r1, r2 = np.frexp(x) assert eq(l1, r1) assert eq(l2, r2) l1, l2 = da.modf(a) r1, r2 = np.modf(x) assert eq(l1, r1) assert eq(l2, r2) assert eq(da.around(a, -1), np.around(x, -1))
def _estimate_poissonian_noise_variance(dc, gain_factor, gain_offset, correlation_factor): variance = (dc * gain_factor + gain_offset) * correlation_factor # The lower bound of the variance is the gaussian noise. variance = da.clip(variance, gain_offset * correlation_factor, np.inf) return variance