def test_elemwise_consistent_names():
a = da.from_array(np.arange(5, dtype='f4'), chunks=(2, ))
b = da.from_array(np.arange(5, dtype='f4'), chunks=(2, ))
assert same_keys(a + b, a + b)
assert same_keys(a + 2, a + 2)
assert same_keys(da.exp(a), da.exp(a))
assert same_keys(da.exp(a, dtype='f8'), da.exp(a, dtype='f8'))
assert same_keys(da.maximum(a, b), da.maximum(a, b))
def test_elemwise_consistent_names():
a = da.from_array(np.arange(5, dtype='f4'), chunks=(2,))
b = da.from_array(np.arange(5, dtype='f4'), chunks=(2,))
assert same_keys(a + b, a + b)
assert same_keys(a + 2, a + 2)
assert same_keys(da.exp(a), da.exp(a))
assert same_keys(da.exp(a, dtype='f8'), da.exp(a, dtype='f8'))
assert same_keys(da.maximum(a, b), da.maximum(a, b))
def get_atm_variables(mus, muv, phi, height, ah2o, bh2o, ao3, tau):
air_mass = 1.0 / mus + 1 / muv
air_mass = air_mass.where(air_mass <= MAXAIRMASS, -1.0)
tO3 = 1.0
tH2O = 1.0
if ao3 != 0:
tO3 = da.exp(-air_mass * UO3 * ao3)
if bh2o != 0:
if bUseV171:
tH2O = da.exp(-da.exp(ah2o + bh2o * da.log(air_mass * UH2O)))
else:
tH2O = da.exp(-(ah2o * ((air_mass * UH2O)**bh2o)))
# Returns sphalb, rhoray, TtotraytH2O, tOG
return atm_variables_finder(mus, muv, phi, height, tau, tO3, tH2O,
TAUSTEP4SPHALB)
def make_classification(n_samples=100, n_features=20, n_informative=2,
n_redundant=2, n_repeated=0, n_classes=2,
n_clusters_per_class=2, weights=None, flip_y=0.01,
class_sep=1.0, hypercube=True, shift=0.0, scale=1.0,
shuffle=True, random_state=None, chunks=None):
chunks = da.core.normalize_chunks(chunks, (n_samples, n_features))
_check_axis_partitioning(chunks, n_features)
if n_classes != 2:
raise NotImplementedError("n_classes != 2 is not yet supported.")
rng = dask_ml.utils.check_random_state(random_state)
X = rng.normal(0, 1, size=(n_samples, n_features),
chunks=chunks)
informative_idx = rng.choice(n_features, n_informative,
chunks=n_informative)
beta = (rng.random(n_features, chunks=n_features) - 1) * scale
informative_idx, beta = dask.compute(informative_idx, beta)
z0 = X[:, informative_idx].dot(beta[informative_idx])
y = rng.random(z0.shape, chunks=chunks[0]) < 1 / (1 + da.exp(-z0))
y = y.astype(int)
return X, y
def black_scholes(nopt, price, strike, t, rate, vol, schd=None):
mr = -rate
sig_sig_two = vol * vol * 2
P = price
S = strike
T = t
a = log(P / S)
b = T * mr
z = T * sig_sig_two
c = 0.25 * z
y = da.map_blocks(invsqrt, z)
w1 = (a - b + c) * y
w2 = (a - b - c) * y
d1 = 0.5 + 0.5 * da.map_blocks(erf, w1)
d2 = 0.5 + 0.5 * da.map_blocks(erf, w2)
Se = exp(b) * S
call = P * d1 - Se * d2
put = call - P + Se
return da.compute(da.stack((put, call)), get=schd)
def gaussian_kernel(x: Union[np.ndarray, da.array],
y: Union[np.ndarray, da.array],
sigma: np.ndarray = None) -> Union[np.ndarray, da.array]:
"""
Gaussian kernel between samples of x and y. A sum of kernels is computed
for multiple values of sigma.
Parameters
----------
x
Batch of instances of shape [Nx, features].
y
Batch of instances of shape [Ny, features].
sigma
Array with values of the kernel width sigma.
chunks
Chunk sizes for x and y when using dask to compute the pairwise distances.
Returns
-------
Array [Nx, Ny] of the kernel.
"""
beta = 1. / (2. * np.expand_dims(sigma, 1)) # [Ns,1]
dist = distance.pairwise_distance(x, y) # [Nx,Ny]
s = beta @ dist.reshape(1, -1) # [Ns,1]*[1,Nx*Ny]=[Ns,Nx*Ny]
s = np.exp(-s) if isinstance(s, np.ndarray) else da.exp(-s)
return s.sum(axis=0).reshape(dist.shape) # [Nx,Ny]
def black_scholes_numpy_mod(nopt, price, strike, t, rate, vol):
mr = -rate
sig_sig_two = vol * vol * 2
P = price
S = strike
T = t
a = log(P / S)
b = T * mr
z = T * sig_sig_two
c = 0.25 * z
y = invsqrt(z)
w1 = (a - b + c) * y
w2 = (a - b - c) * y
d1 = 0.5 + 0.5 * erf(w1)
d2 = 0.5 + 0.5 * erf(w2)
Se = exp(b) * S
call = P * d1 - Se * d2
put = call - P + Se
return np.stack((call, put))
def atm_variables_finder(mus,
muv,
phi,
height,
tau,
tO3,
tH2O,
taustep4sphalb,
tO2=1.0):
tau_step = da.linspace(taustep4sphalb,
MAXNUMSPHALBVALUES * taustep4sphalb,
MAXNUMSPHALBVALUES,
chunks=int(MAXNUMSPHALBVALUES / 2))
sphalb0 = csalbr(tau_step)
taur = tau * da.exp(-height / SCALEHEIGHT)
rhoray, trdown, trup = chand(phi, muv, mus, taur)
if isinstance(height, xr.DataArray):
def _sphalb_index(index_arr, sphalb0):
# FIXME: if/when dask can support lazy index arrays then remove this
return sphalb0[index_arr]
sphalb = da.map_blocks(_sphalb_index, (taur / taustep4sphalb +
0.5).astype(np.int32).data,
sphalb0.compute(),
dtype=sphalb0.dtype)
else:
sphalb = sphalb0[(taur / taustep4sphalb + 0.5).astype(np.int32)]
Ttotrayu = ((2 / 3. + muv) + (2 / 3. - muv) * trup) / (4 / 3. + taur)
Ttotrayd = ((2 / 3. + mus) + (2 / 3. - mus) * trdown) / (4 / 3. + taur)
TtotraytH2O = Ttotrayu * Ttotrayd * tH2O
tOG = tO3 * tO2
return sphalb, rhoray, TtotraytH2O, tOG
def smooth(D, y, sigma):
"""Given a matrix ``D`` defining the squared distance between training and
prediction points, and a matrix or vector y defining one or more lets of labels at
the training points, return predicitons using an RBF kernel.
Parameters
----------
D : array-like
Squared distance matrix. $D_{i,j}$ defines the squared distance between training
point ``i`` and prediction point ``j``.
y : array-like
Labels for training points. May be 1d if a single prediction task or 2d if >1
prediction tasks.
sigma : int
The length scale for RBF smoothing. i.e. the weight of a training point is
proportional to $exp((-.5/sigma**2)*D_{i,j})$
Returns
-------
smoothed_predictions : array-like
Predicted labels
"""
y = solve_functions.y_to_matrix(y)
# get RBF smoothing matrix
S = da.exp((-0.5 / sigma**2) * D)
# if sigma is small enough, weights could turn to 0, so we reset those to
# just guess the average of the training set
S = da.where(S.sum(axis=1).reshape(-1, 1) > 0, S, 1)
smoothed_predictions = S.dot(y) / S.sum(axis=1).reshape(-1, 1)
return smoothed_predictions
def logsumexp(arr, axis=0):
"""Computes the sum of arr assuming arr is in the log domain.
Returns log(sum(exp(arr))) while minimizing the possibility of
over/underflow.
Examples
--------
>>> import numpy as np
>>> from sklearn.utils.extmath import logsumexp
>>> a = np.arange(10)
>>> np.log(np.sum(np.exp(a)))
9.4586297444267107
>>> logsumexp(a)
9.4586297444267107
"""
if axis == 0:
pass
elif axis == 1:
arr = arr.T
else:
raise NotImplementedError
# Use the max to normalize, as with the log this is what accumulates
# the less errors
vmax = arr.max(axis=0)
out = da.log(da.sum(da.exp(arr - vmax), axis=0))
out += vmax
return out
def gradient(X, y, max_steps=100):
N, M = X.shape
firstBacktrackMult = 0.1
nextBacktrackMult = 0.5
armijoMult = 0.1
stepGrowth = 1.25
stepSize = 1.0
recalcRate = 10
backtrackMult = firstBacktrackMult
beta = np.zeros(M)
print('## -f |df/f| |dx/x| step')
print('----------------------------------------------')
for k in range(max_steps):
# Compute the gradient
if k % recalcRate == 0:
Xbeta = X.dot(beta)
eXbeta = da.exp(Xbeta)
func = da.log1p(eXbeta).sum() - y.dot(Xbeta)
e1 = eXbeta + 1.0
gradient = X.T.dot(eXbeta / e1 - y)
steplen = (gradient**2).sum()**0.5
Xgradient = X.dot(gradient)
Xbeta, eXbeta, func, gradient, steplen, Xgradient = da.compute(
Xbeta, eXbeta, func, gradient, steplen, Xgradient)
obeta = beta
oXbeta = Xbeta
# Compute the step size
lf = func
for ii in range(100):
beta = obeta - stepSize * gradient
if ii and np.array_equal(beta, obeta):
stepSize = 0
break
Xbeta = oXbeta - stepSize * Xgradient
# This prevents overflow
if np.all(Xbeta < 700):
eXbeta = np.exp(Xbeta)
func = np.sum(np.log1p(eXbeta)) - np.dot(y, Xbeta)
df = lf - func
if df >= armijoMult * stepSize * steplen**2:
break
stepSize *= backtrackMult
if stepSize == 0:
print('No more progress')
break
df /= max(func, lf)
db = stepSize * steplen / (np.linalg.norm(beta) + stepSize * steplen)
print('%2d %.6e %9.2e %.2e %.1e' % (k + 1, func, df, db, stepSize))
if df < 1e-14:
print('Converged')
break
stepSize *= stepGrowth
backtrackMult = nextBacktrackMult
return beta
def rbf_kernel(X, Y=None, gamma=None):
X, Y = check_pairwise_arrays(X, Y)
if gamma is None:
gamma = 1.0 / X.shape[1]
K = euclidean_distances(X, Y, squared=True)
K = da.exp(-gamma * K)
return K
def make_classification(n_samples=1000, n_features=100, n_informative=2, scale=1.0,
chunksize=100):
X = da.random.normal(0, 1, size=(n_samples, n_features),
chunks=(chunksize, n_features))
informative_idx = np.random.choice(n_features, n_informative)
beta = (np.random.random(n_features) - 1) * scale
z0 = X[:, informative_idx].dot(beta[informative_idx])
y = da.random.random(z0.shape, chunks=(chunksize,)) < 1 / (1 + da.exp(-z0))
return X, y
def get_atm_variables_abi(mus, muv, phi, height, G_O3, G_H2O, G_O2, ah2o, ao2,
ao3, tau):
tO3 = 1.0
tH2O = 1.0
if ao3 != 0:
tO3 = da.exp(-G_O3 * ao3)
if ah2o != 0:
tH2O = da.exp(-G_H2O * ah2o)
tO2 = da.exp(-G_O2 * ao2)
# Returns sphalb, rhoray, TtotraytH2O, tOG.
return atm_variables_finder(mus,
muv,
phi,
height,
tau,
tO3,
tH2O,
TAUSTEP4SPHALB_ABI,
tO2=tO2)
def rbf_kernel(X: ArrayLike,
Y: Optional[ArrayLike] = None,
gamma: Optional[float] = None) -> ArrayLike:
X, Y = check_pairwise_arrays(X, Y)
if gamma is None:
gamma = 1.0 / X.shape[1]
K = euclidean_distances(X, Y, squared=True)
K = da.exp(-gamma * K)
return K
def make_counts(
n_samples=1000,
n_features=100,
n_informative=2,
scale=1.0,
chunks=100,
random_state=None,
):
"""
Generate a dummy dataset for modeling count data.
Parameters
----------
n_samples : int
number of rows in the output array
n_features : int
number of columns (features) in the output array
n_informative : int
number of features that are correlated with the outcome
scale : float
Scale the true coefficient array by this
chunks : int
Number of rows per dask array block.
random_state : int, RandomState instance or None (default)
Determines random number generation for dataset creation. Pass an int
for reproducible output across multiple function calls.
See :term:`Glossary <random_state>`.
Returns
-------
X : dask.array, size ``(n_samples, n_features)``
y : dask.array, size ``(n_samples,)``
array of non-negative integer-valued data
Examples
--------
>>> X, y = make_counts()
"""
rng = dask_ml.utils.check_random_state(random_state)
X = rng.normal(0,
1,
size=(n_samples, n_features),
chunks=(chunks, n_features))
informative_idx = rng.choice(n_features,
n_informative,
chunks=n_informative)
beta = (rng.random(n_features, chunks=n_features) - 1) * scale
informative_idx, beta = dask.compute(informative_idx, beta)
z0 = X[:, informative_idx].dot(beta[informative_idx])
rate = da.exp(z0)
y = rng.poisson(rate, size=1, chunks=(chunks, ))
return X, y
def gradient_descent(X, y, max_steps=100, tol=1e-14):
'''Michael Grant's implementation of Gradient Descent.'''
n, p = X.shape
firstBacktrackMult = 0.1
nextBacktrackMult = 0.5
armijoMult = 0.1
stepGrowth = 1.25
stepSize = 1.0
recalcRate = 10
backtrackMult = firstBacktrackMult
beta = np.zeros(p)
y_local = y.compute()
for k in range(max_steps):
# how necessary is this recalculation?
if k % recalcRate == 0:
Xbeta = X.dot(beta)
eXbeta = da.exp(Xbeta)
func = da.log1p(eXbeta).sum() - y.dot(Xbeta)
e1 = eXbeta + 1.0
gradient = X.T.dot(eXbeta / e1 - y)
Xgradient = X.dot(gradient)
Xbeta, eXbeta, func, gradient, Xgradient = da.compute(
Xbeta, eXbeta, func, gradient, Xgradient)
# backtracking line search
lf = func
stepSize, beta, Xbeta, func = compute_stepsize(beta, gradient,
Xbeta, Xgradient,
y_local, func,
**{
'backtrackMult': backtrackMult,
'armijoMult': armijoMult,
'stepSize': stepSize})
if stepSize == 0:
print('No more progress')
break
# necessary for gradient computation
eXbeta = exp(Xbeta)
df = lf - func
df /= max(func, lf)
if df < tol:
print('Converged')
break
stepSize *= stepGrowth
backtrackMult = nextBacktrackMult
return beta
def predict_proba(self, X):
"""
Return probability estimates for the test vector X.
Parameters
----------
X : array-like, shape = [n_samples, n_features]
Returns
-------
C : array-like, shape = [n_samples, n_classes]
Returns the probability of the samples for each class in
the model. The columns correspond to the classes in sorted
order, as they appear in the attribute `classes_`.
"""
return da.exp(self.predict_log_proba(X))
def make_classification(n_samples=1000,
n_features=100,
n_informative=2,
scale=1.0,
chunksize=100,
is_sparse=False):
"""
Generate a dummy dataset for classification tasks.
Parameters
----------
n_samples : int
number of rows in the output array
n_features : int
number of columns (features) in the output array
n_informative : int
number of features that are correlated with the outcome
scale : float
Scale the true coefficient array by this
chunksize : int
Number of rows per dask array block.
is_sparse: bool
Return a sparse matrix
Returns
-------
X : dask.array, size ``(n_samples, n_features)``
y : dask.array, size ``(n_samples,)``
boolean-valued array
Examples
--------
>>> X, y = make_classification()
>>> X
dask.array<da.random.normal, shape=(1000, 100), dtype=float64, chunksize=(100, 100)>
>>> y
dask.array<lt, shape=(1000,), dtype=bool, chunksize=(100,)>
"""
X = da.random.normal(0,
1,
size=(n_samples, n_features),
chunks=(chunksize, n_features))
if is_sparse:
X = X.map_blocks(sparse.COO)
informative_idx = np.random.choice(n_features, n_informative)
beta = (np.random.random(n_features) - 1) * scale
z0 = X[:, informative_idx].dot(beta[informative_idx])
y = da.random.random(z0.shape,
chunks=(chunksize, )) < 1 / (1 + da.exp(-z0))
return X, y
def gram_rbf(X, threshold=1.0):
if type(X) == torch.Tensor:
dot_products = X @ X.t()
sq_norms = dot_products.diag()
sq_distances = -2*dot_products + sq_norms[:,None] + sq_norms[None,:]
sq_median_distance = sq_distances.median()
return torch.exp(-sq_distances / (2*threshold**2 * sq_median_distance))
elif type(X) == da.Array:
dot_products = X @ X.T
sq_norms = da.diag(dot_products)
sq_distances = -2*dot_products + sq_norms[:,None] + sq_norms[None,:]
sq_median_distance = da.percentile(sq_distances.ravel(), 50)
return da.exp((-sq_distances / (2*threshold**2 * sq_median_distance)))
else:
raise ValueError
def test_dtype_complex():
x = np.arange(24).reshape((4, 6)).astype('f4')
y = np.arange(24).reshape((4, 6)).astype('i8')
z = np.arange(24).reshape((4, 6)).astype('i2')
a = da.from_array(x, chunks=(2, 3))
b = da.from_array(y, chunks=(2, 3))
c = da.from_array(z, chunks=(2, 3))
def eq(a, b):
return (isinstance(a, np.dtype) and
isinstance(b, np.dtype) and
str(a) == str(b))
assert eq(a._dtype, x.dtype)
assert eq(b._dtype, y.dtype)
assert eq((a + 1)._dtype, (x + 1).dtype)
assert eq((a + b)._dtype, (x + y).dtype)
assert eq(a.T._dtype, x.T.dtype)
assert eq(a[:3]._dtype, x[:3].dtype)
assert eq((a.dot(b.T))._dtype, (x.dot(y.T)).dtype)
assert eq(stack([a, b])._dtype, np.vstack([x, y]).dtype)
assert eq(concatenate([a, b])._dtype, np.concatenate([x, y]).dtype)
assert eq(b.std()._dtype, y.std().dtype)
assert eq(c.sum()._dtype, z.sum().dtype)
assert eq(a.min()._dtype, a.min().dtype)
assert eq(b.std()._dtype, b.std().dtype)
assert eq(a.argmin(axis=0)._dtype, a.argmin(axis=0).dtype)
assert eq(da.sin(c)._dtype, np.sin(z).dtype)
assert eq(da.exp(b)._dtype, np.exp(y).dtype)
assert eq(da.floor(a)._dtype, np.floor(x).dtype)
assert eq(da.isnan(b)._dtype, np.isnan(y).dtype)
with ignoring(ImportError):
assert da.isnull(b)._dtype == 'bool'
assert da.notnull(b)._dtype == 'bool'
x = np.array([('a', 1)], dtype=[('text', 'S1'), ('numbers', 'i4')])
d = da.from_array(x, chunks=(1,))
assert eq(d['text']._dtype, x['text'].dtype)
assert eq(d[['numbers', 'text']]._dtype, x[['numbers', 'text']].dtype)
def test_dtype_complex():
x = np.arange(24).reshape((4, 6)).astype('f4')
y = np.arange(24).reshape((4, 6)).astype('i8')
z = np.arange(24).reshape((4, 6)).astype('i2')
a = da.from_array(x, chunks=(2, 3))
b = da.from_array(y, chunks=(2, 3))
c = da.from_array(z, chunks=(2, 3))
def eq(a, b):
return (isinstance(a, np.dtype) and isinstance(b, np.dtype)
and str(a) == str(b))
assert eq(a._dtype, x.dtype)
assert eq(b._dtype, y.dtype)
assert eq((a + 1)._dtype, (x + 1).dtype)
assert eq((a + b)._dtype, (x + y).dtype)
assert eq(a.T._dtype, x.T.dtype)
assert eq(a[:3]._dtype, x[:3].dtype)
assert eq((a.dot(b.T))._dtype, (x.dot(y.T)).dtype)
assert eq(stack([a, b])._dtype, np.vstack([x, y]).dtype)
assert eq(concatenate([a, b])._dtype, np.concatenate([x, y]).dtype)
assert eq(b.std()._dtype, y.std().dtype)
assert eq(c.sum()._dtype, z.sum().dtype)
assert eq(a.min()._dtype, a.min().dtype)
assert eq(b.std()._dtype, b.std().dtype)
assert eq(a.argmin(axis=0)._dtype, a.argmin(axis=0).dtype)
assert eq(da.sin(c)._dtype, np.sin(z).dtype)
assert eq(da.exp(b)._dtype, np.exp(y).dtype)
assert eq(da.floor(a)._dtype, np.floor(x).dtype)
assert eq(da.isnan(b)._dtype, np.isnan(y).dtype)
with ignoring(ImportError):
assert da.isnull(b)._dtype == 'bool'
assert da.notnull(b)._dtype == 'bool'
x = np.array([('a', 1)], dtype=[('text', 'S1'), ('numbers', 'i4')])
d = da.from_array(x, chunks=(1, ))
assert eq(d['text']._dtype, x['text'].dtype)
assert eq(d[['numbers', 'text']]._dtype, x[['numbers', 'text']].dtype)
def matern32(coords, lambda0):
""" Matern 3/2 covariance kernel.
Parameters
----------
coords: (n_pts, n_dims) dask.array or Future
Point coordinates.
Returns
-------
covs: (n_pts, n_pts) delayed dask.array
Pairwise covariance matrix.
"""
dists = dask_distance.euclidean(coords)
res = da.multiply(
1 + (np.sqrt(3) / lambda0) * dists,
da.exp(-(np.sqrt(3) / lambda0) * dists))
return res
def make_counts(n_samples=1000,
n_features=100,
n_informative=2,
scale=1.0,
chunks=100):
"""
Generate a dummy dataset for modeling count data.
Parameters
----------
n_samples : int
number of rows in the output array
n_features : int
number of columns (features) in the output array
n_informative : int
number of features that are correlated with the outcome
scale : float
Scale the true coefficient array by this
chunks : int
Number of rows per dask array block.
Returns
-------
X : dask.array, size ``(n_samples, n_features)``
y : dask.array, size ``(n_samples,)``
array of non-negative integer-valued data
Examples
--------
>>> X, y = make_counts()
"""
X = da.random.normal(0,
1,
size=(n_samples, n_features),
chunks=(chunks, n_features))
informative_idx = np.random.choice(n_features, n_informative)
beta = (np.random.random(n_features) - 1) * scale
z0 = X[:, informative_idx].dot(beta[informative_idx])
rate = da.exp(z0)
y = da.random.poisson(rate, size=1, chunks=(chunks, ))
return X, y
def get_harmonics(self, in_sphere):
"""Create the harmonics for this arrangement of sphere pixels"""
# cache_key = "{}:".format(in_sphere.npix)
# if (cache_key in self.harmonics):
# return self.harmonics[cache_key]
harmonic_list = []
p2j = jomega(self.frequency)
# logger.info("pixel areas: {}".format(in_sphere.pixel_areas))
for u, v, w in zip(self.u_arr, self.v_arr, self.w_arr):
harmonic = (da.exp(
p2j *
(u * in_sphere.l + v * in_sphere.m + w * in_sphere.n_minus_1))
* in_sphere.pixel_areas)
assert harmonic.shape[0] == in_sphere.npix
harmonic_list.append(harmonic)
# self.harmonics[cache_key] = harmonic_list
# assert(harmonic_list[0].shape[0] == in_sphere.npix)
return harmonic_list
def fourier_shift(image, shift, n=-1, axis=-1):
"""
Multi-dimensional fourier shift filter.
The array is multiplied with the fourier transform of a shift operation.
Parameters
----------
image : array_like
The input image.
shift : float or sequence
The size of the box used for filtering.
If a float, `shift` is the same for all axes. If a sequence, `shift`
has to contain one value for each axis.
n : int, optional
If `n` is negative (default), then the image is assumed to be the
result of a complex fft.
If `n` is larger than or equal to zero, the image is assumed to be the
result of a real fft, and `n` gives the length of the array before
transformation along the real transform direction.
axis : int, optional
The axis of the real transform.
Returns
-------
fourier_shift : Dask Array
Examples
--------
>>> from scipy import ndimage, misc
>>> import matplotlib.pyplot as plt
>>> import numpy.fft
>>> fig, (ax1, ax2) = plt.subplots(1, 2)
>>> plt.gray() # show the filtered result in grayscale
>>> ascent = misc.ascent()
>>> image = numpy.fft.fft2(ascent)
>>> result = ndimage.fourier_shift(image, shift=200)
>>> result = numpy.fft.ifft2(result)
>>> ax1.imshow(ascent)
>>> ax2.imshow(result.real) # the imaginary part is an artifact
>>> plt.show()
"""
if issubclass(image.dtype.type, numbers.Real):
image = image.astype(complex)
# Validate and normalize arguments
image, shift, n, axis = _utils._norm_args(image, shift, n=n, axis=axis)
# Constants with type converted
J = image.dtype.type(1j)
# Get the grid of frequencies
ang_freq_grid = _utils._get_ang_freq_grid(image.shape,
chunks=image.chunks,
n=n,
axis=axis,
dtype=shift.dtype)
# Apply shift
result = image.copy()
for ax, f in enumerate(ang_freq_grid):
phase_shift = da.exp((-J) * shift[ax] * f)
phase_shift = _utils._reshape_nd(phase_shift, ndim=image.ndim, axis=ax)
result *= phase_shift
return result
def test_arithmetic():
x = np.arange(5).astype('f4') + 2
y = np.arange(5).astype('i8') + 2
z = np.arange(5).astype('i4') + 2
a = da.from_array(x, chunks=(2,))
b = da.from_array(y, chunks=(2,))
c = da.from_array(z, chunks=(2,))
assert eq(a + b, x + y)
assert eq(a * b, x * y)
assert eq(a - b, x - y)
assert eq(a / b, x / y)
assert eq(b & b, y & y)
assert eq(b | b, y | y)
assert eq(b ^ b, y ^ y)
assert eq(a // b, x // y)
assert eq(a ** b, x ** y)
assert eq(a % b, x % y)
assert eq(a > b, x > y)
assert eq(a < b, x < y)
assert eq(a >= b, x >= y)
assert eq(a <= b, x <= y)
assert eq(a == b, x == y)
assert eq(a != b, x != y)
assert eq(a + 2, x + 2)
assert eq(a * 2, x * 2)
assert eq(a - 2, x - 2)
assert eq(a / 2, x / 2)
assert eq(b & True, y & True)
assert eq(b | True, y | True)
assert eq(b ^ True, y ^ True)
assert eq(a // 2, x // 2)
assert eq(a ** 2, x ** 2)
assert eq(a % 2, x % 2)
assert eq(a > 2, x > 2)
assert eq(a < 2, x < 2)
assert eq(a >= 2, x >= 2)
assert eq(a <= 2, x <= 2)
assert eq(a == 2, x == 2)
assert eq(a != 2, x != 2)
assert eq(2 + b, 2 + y)
assert eq(2 * b, 2 * y)
assert eq(2 - b, 2 - y)
assert eq(2 / b, 2 / y)
assert eq(True & b, True & y)
assert eq(True | b, True | y)
assert eq(True ^ b, True ^ y)
assert eq(2 // b, 2 // y)
assert eq(2 ** b, 2 ** y)
assert eq(2 % b, 2 % y)
assert eq(2 > b, 2 > y)
assert eq(2 < b, 2 < y)
assert eq(2 >= b, 2 >= y)
assert eq(2 <= b, 2 <= y)
assert eq(2 == b, 2 == y)
assert eq(2 != b, 2 != y)
assert eq(-a, -x)
assert eq(abs(a), abs(x))
assert eq(~(a == b), ~(x == y))
assert eq(~(a == b), ~(x == y))
assert eq(da.logaddexp(a, b), np.logaddexp(x, y))
assert eq(da.logaddexp2(a, b), np.logaddexp2(x, y))
assert eq(da.exp(b), np.exp(y))
assert eq(da.log(a), np.log(x))
assert eq(da.log10(a), np.log10(x))
assert eq(da.log1p(a), np.log1p(x))
assert eq(da.expm1(b), np.expm1(y))
assert eq(da.sqrt(a), np.sqrt(x))
assert eq(da.square(a), np.square(x))
assert eq(da.sin(a), np.sin(x))
assert eq(da.cos(b), np.cos(y))
assert eq(da.tan(a), np.tan(x))
assert eq(da.arcsin(b/10), np.arcsin(y/10))
assert eq(da.arccos(b/10), np.arccos(y/10))
assert eq(da.arctan(b/10), np.arctan(y/10))
assert eq(da.arctan2(b*10, a), np.arctan2(y*10, x))
assert eq(da.hypot(b, a), np.hypot(y, x))
assert eq(da.sinh(a), np.sinh(x))
assert eq(da.cosh(b), np.cosh(y))
assert eq(da.tanh(a), np.tanh(x))
assert eq(da.arcsinh(b*10), np.arcsinh(y*10))
assert eq(da.arccosh(b*10), np.arccosh(y*10))
assert eq(da.arctanh(b/10), np.arctanh(y/10))
assert eq(da.deg2rad(a), np.deg2rad(x))
assert eq(da.rad2deg(a), np.rad2deg(x))
assert eq(da.logical_and(a < 1, b < 4), np.logical_and(x < 1, y < 4))
assert eq(da.logical_or(a < 1, b < 4), np.logical_or(x < 1, y < 4))
assert eq(da.logical_xor(a < 1, b < 4), np.logical_xor(x < 1, y < 4))
assert eq(da.logical_not(a < 1), np.logical_not(x < 1))
assert eq(da.maximum(a, 5 - a), np.maximum(a, 5 - a))
assert eq(da.minimum(a, 5 - a), np.minimum(a, 5 - a))
assert eq(da.fmax(a, 5 - a), np.fmax(a, 5 - a))
assert eq(da.fmin(a, 5 - a), np.fmin(a, 5 - a))
assert eq(da.isreal(a + 1j * b), np.isreal(x + 1j * y))
assert eq(da.iscomplex(a + 1j * b), np.iscomplex(x + 1j * y))
assert eq(da.isfinite(a), np.isfinite(x))
assert eq(da.isinf(a), np.isinf(x))
assert eq(da.isnan(a), np.isnan(x))
assert eq(da.signbit(a - 3), np.signbit(x - 3))
assert eq(da.copysign(a - 3, b), np.copysign(x - 3, y))
assert eq(da.nextafter(a - 3, b), np.nextafter(x - 3, y))
assert eq(da.ldexp(c, c), np.ldexp(z, z))
assert eq(da.fmod(a * 12, b), np.fmod(x * 12, y))
assert eq(da.floor(a * 0.5), np.floor(x * 0.5))
assert eq(da.ceil(a), np.ceil(x))
assert eq(da.trunc(a / 2), np.trunc(x / 2))
assert eq(da.degrees(b), np.degrees(y))
assert eq(da.radians(a), np.radians(x))
assert eq(da.rint(a + 0.3), np.rint(x + 0.3))
assert eq(da.fix(a - 2.5), np.fix(x - 2.5))
assert eq(da.angle(a + 1j), np.angle(x + 1j))
assert eq(da.real(a + 1j), np.real(x + 1j))
assert eq((a + 1j).real, np.real(x + 1j))
assert eq(da.imag(a + 1j), np.imag(x + 1j))
assert eq((a + 1j).imag, np.imag(x + 1j))
assert eq(da.conj(a + 1j * b), np.conj(x + 1j * y))
assert eq((a + 1j * b).conj(), (x + 1j * y).conj())
assert eq(da.clip(b, 1, 4), np.clip(y, 1, 4))
assert eq(da.fabs(b), np.fabs(y))
assert eq(da.sign(b - 2), np.sign(y - 2))
l1, l2 = da.frexp(a)
r1, r2 = np.frexp(x)
assert eq(l1, r1)
assert eq(l2, r2)
l1, l2 = da.modf(a)
r1, r2 = np.modf(x)
assert eq(l1, r1)
assert eq(l2, r2)
assert eq(da.around(a, -1), np.around(x, -1))
def softmax_dask(x):
x = da.from_array(x, chunks=int(x.shape[0] / CPU_COUNT), name='x')
e_x = da.exp(x - x.max())
out = e_x / e_x.sum()
normalized = out.compute()
return normalized
def pressure(x, y, t, t0=10000., U=50., a=200000., p0=2000., x0=0.):
return (p0 * (1. - da.exp(-t / t0)) * da.exp(-((x - x0)**2. +
(y - U * t)**2.) / a**2.))
def _set_amplitude(self, amplitude):
if isinstance(amplitude, BaseSignal):
amplitude = amplitude.data.real
self.data = amplitude * da.exp(1j * da.angle(self.data))
self.events.data_changed.trigger(self)
def _set_phase(self, phase):
if isinstance(phase, BaseSignal):
phase = phase.data.real
self.data = abs(self.data) * \
da.exp(1j * phase)
self.events.data_changed.trigger(self)
def _set_amplitude(self, amplitude):
if isinstance(amplitude, BaseSignal):
amplitude = amplitude.data.real
self.data = amplitude * da.exp(1j * da.angle(self.data))
self.events.data_changed.trigger(self)
def _set_phase(self, phase):
if isinstance(phase, BaseSignal):
phase = phase.data.real
self.data = da.numpy_compat.builtins.abs(self.data) * \
da.exp(1j * phase)
self.events.data_changed.trigger(self)
def exp(A):
return da.exp(A)
def make_dirty(self):
print("Making dirty", file=log)
dirty = da.zeros((self.nband, self.nx, self.ny),
dtype=np.float32,
chunks=(1, self.nx, self.ny),
name=False)
dirties = []
for ims in self.ms:
xds = xds_from_ms(ims,
group_cols=('FIELD_ID', 'DATA_DESC_ID'),
chunks=self.chunks[ims],
columns=self.columns)
# subtables
ddids = xds_from_table(ims + "::DATA_DESCRIPTION")
fields = xds_from_table(ims + "::FIELD", group_cols="__row__")
spws = xds_from_table(ims + "::SPECTRAL_WINDOW",
group_cols="__row__")
pols = xds_from_table(ims + "::POLARIZATION", group_cols="__row__")
# subtable data
ddids = dask.compute(ddids)[0]
fields = dask.compute(fields)[0]
spws = dask.compute(spws)[0]
pols = dask.compute(pols)[0]
for ds in xds:
field = fields[ds.FIELD_ID]
radec = field.PHASE_DIR.data.squeeze()
if not np.array_equal(radec, self.radec):
continue
spw = ds.DATA_DESC_ID # this is not correct, need to use spw
freq_bin_idx = self.freq_bin_idx[ims][spw]
freq_bin_counts = self.freq_bin_counts[ims][spw]
freq = self.freq[ims][spw]
freq_chunk = freq_bin_counts[0].compute()
uvw = ds.UVW.data
data = getattr(ds, self.data_column).data
dataxx = data[:, :, 0]
datayy = data[:, :, -1]
weights = getattr(ds, self.weight_column).data
if len(weights.shape) < 3:
weights = da.broadcast_to(weights[:, None, :],
data.shape,
chunks=data.chunks)
if self.imaging_weight_column is not None:
imaging_weights = getattr(ds,
self.imaging_weight_column).data
if len(imaging_weights.shape) < 3:
imaging_weights = da.broadcast_to(
imaging_weights[:, None, :],
data.shape,
chunks=data.chunks)
weightsxx = imaging_weights[:, :, 0] * weights[:, :, 0]
weightsyy = imaging_weights[:, :, -1] * weights[:, :, -1]
else:
weightsxx = weights[:, :, 0]
weightsyy = weights[:, :, -1]
# apply adjoint of mueller term.
# Phases modify data amplitudes modify weights.
if self.mueller_column is not None:
mueller = getattr(ds, self.mueller_column).data
dataxx *= da.exp(-1j * da.angle(mueller[:, :, 0]))
datayy *= da.exp(-1j * da.angle(mueller[:, :, -1]))
weightsxx *= da.absolute(mueller[:, :, 0])
weightsyy *= da.absolute(mueller[:, :, -1])
# weighted sum corr to Stokes I
weights = weightsxx + weightsyy
data = (weightsxx * dataxx + weightsyy * datayy)
# TODO - turn off this stupid warning
data = da.where(weights, data / weights, 0.0j)
# only keep data where both corrs are unflagged
flag = getattr(ds, self.flag_column).data
flagxx = flag[:, :, 0]
flagyy = flag[:, :, -1]
# ducc0 convention uses uint8 mask not flag
flag = ~(flagxx | flagyy)
dirty = vis2im(uvw,
freq,
data,
freq_bin_idx,
freq_bin_counts,
self.nx,
self.ny,
self.cell,
weights=weights,
flag=flag.astype(np.uint8),
nthreads=self.nthreads,
epsilon=self.epsilon,
do_wstacking=self.do_wstacking,
double_accum=True)
dirties.append(dirty)
dirties = dask.compute(dirties, scheduler='single-threaded')[0]
return accumulate_dirty(dirties, self.nband,
self.band_mapping).astype(self.real_type)
def chand(phi, muv, mus, taur):
# FROM FUNCTION CHAND
# phi: azimuthal difference between sun and observation in degree
# (phi=0 in backscattering direction)
# mus: cosine of the sun zenith angle
# muv: cosine of the observation zenith angle
# taur: molecular optical depth
# rhoray: molecular path reflectance
# constant xdep: depolarization factor (0.0279)
# xfd = (1-xdep/(2-xdep)) / (1 + 2*xdep/(2-xdep)) = 2 * (1 - xdep) / (2 + xdep) = 0.958725775
# */
xfd = 0.958725775
xbeta2 = 0.5
# float pl[5];
# double fs01, fs02, fs0, fs1, fs2;
as0 = [
0.33243832, 0.16285370, -0.30924818, -0.10324388, 0.11493334,
-6.777104e-02, 1.577425e-03, -1.240906e-02, 3.241678e-02, -3.503695e-02
]
as1 = [0.19666292, -5.439061e-02]
as2 = [0.14545937, -2.910845e-02]
# float phios, xcos1, xcos2, xcos3;
# float xph1, xph2, xph3, xitm1, xitm2;
# float xlntaur, xitot1, xitot2, xitot3;
# int i, ib;
xph1 = 1.0 + (3.0 * mus * mus - 1.0) * (3.0 * muv * muv - 1.0) * xfd / 8.0
xph2 = -xfd * xbeta2 * 1.5 * mus * muv * da.sqrt(
1.0 - mus * mus) * da.sqrt(1.0 - muv * muv)
xph3 = xfd * xbeta2 * 0.375 * (1.0 - mus * mus) * (1.0 - muv * muv)
# pl[0] = 1.0
# pl[1] = mus + muv
# pl[2] = mus * muv
# pl[3] = mus * mus + muv * muv
# pl[4] = mus * mus * muv * muv
fs01 = as0[0] + (mus + muv) * as0[1] + (mus * muv) * as0[2] + (
mus * mus + muv * muv) * as0[3] + (mus * mus * muv * muv) * as0[4]
fs02 = as0[5] + (mus + muv) * as0[6] + (mus * muv) * as0[7] + (
mus * mus + muv * muv) * as0[8] + (mus * mus * muv * muv) * as0[9]
# for (i = 0; i < 5; i++) {
# fs01 += (double) (pl[i] * as0[i]);
# fs02 += (double) (pl[i] * as0[5 + i]);
# }
# for refl, (ah2o, bh2o, ao3, tau) in zip(reflectance_bands, coefficients):
# ib = find_coefficient_index(center_wl)
# if ib is None:
# raise ValueError("Can't handle band with wavelength '{}'".format(center_wl))
xlntaur = da.log(taur)
fs0 = fs01 + fs02 * xlntaur
fs1 = as1[0] + xlntaur * as1[1]
fs2 = as2[0] + xlntaur * as2[1]
del xlntaur, fs01, fs02
trdown = da.exp(-taur / mus)
trup = da.exp(-taur / muv)
xitm1 = (1.0 - trdown * trup) / 4.0 / (mus + muv)
xitm2 = (1.0 - trdown) * (1.0 - trup)
xitot1 = xph1 * (xitm1 + xitm2 * fs0)
xitot2 = xph2 * (xitm1 + xitm2 * fs1)
xitot3 = xph3 * (xitm1 + xitm2 * fs2)
del xph1, xph2, xph3, xitm1, xitm2, fs0, fs1, fs2
phios = da.deg2rad(phi + 180.0)
xcos1 = 1.0
xcos2 = da.cos(phios)
xcos3 = da.cos(2.0 * phios)
del phios
rhoray = xitot1 * xcos1 + xitot2 * xcos2 * 2.0 + xitot3 * xcos3 * 2.0
return rhoray, trdown, trup