Exemplo n.º 1
0
def test_elemwise_consistent_names():
    a = da.from_array(np.arange(5, dtype='f4'), chunks=(2, ))
    b = da.from_array(np.arange(5, dtype='f4'), chunks=(2, ))
    assert same_keys(a + b, a + b)
    assert same_keys(a + 2, a + 2)
    assert same_keys(da.exp(a), da.exp(a))
    assert same_keys(da.exp(a, dtype='f8'), da.exp(a, dtype='f8'))
    assert same_keys(da.maximum(a, b), da.maximum(a, b))
Exemplo n.º 2
0
def test_elemwise_consistent_names():
    a = da.from_array(np.arange(5, dtype='f4'), chunks=(2,))
    b = da.from_array(np.arange(5, dtype='f4'), chunks=(2,))
    assert same_keys(a + b, a + b)
    assert same_keys(a + 2, a + 2)
    assert same_keys(da.exp(a), da.exp(a))
    assert same_keys(da.exp(a, dtype='f8'), da.exp(a, dtype='f8'))
    assert same_keys(da.maximum(a, b), da.maximum(a, b))
Exemplo n.º 3
0
def get_atm_variables(mus, muv, phi, height, ah2o, bh2o, ao3, tau):
    air_mass = 1.0 / mus + 1 / muv
    air_mass = air_mass.where(air_mass <= MAXAIRMASS, -1.0)
    tO3 = 1.0
    tH2O = 1.0
    if ao3 != 0:
        tO3 = da.exp(-air_mass * UO3 * ao3)
    if bh2o != 0:
        if bUseV171:
            tH2O = da.exp(-da.exp(ah2o + bh2o * da.log(air_mass * UH2O)))
        else:
            tH2O = da.exp(-(ah2o * ((air_mass * UH2O)**bh2o)))
    # Returns sphalb, rhoray, TtotraytH2O, tOG
    return atm_variables_finder(mus, muv, phi, height, tau, tO3, tH2O,
                                TAUSTEP4SPHALB)
Exemplo n.º 4
0
def make_classification(n_samples=100, n_features=20, n_informative=2,
                        n_redundant=2, n_repeated=0, n_classes=2,
                        n_clusters_per_class=2, weights=None, flip_y=0.01,
                        class_sep=1.0, hypercube=True, shift=0.0, scale=1.0,
                        shuffle=True, random_state=None, chunks=None):
    chunks = da.core.normalize_chunks(chunks, (n_samples, n_features))
    _check_axis_partitioning(chunks, n_features)

    if n_classes != 2:
        raise NotImplementedError("n_classes != 2 is not yet supported.")

    rng = dask_ml.utils.check_random_state(random_state)

    X = rng.normal(0, 1, size=(n_samples, n_features),
                   chunks=chunks)
    informative_idx = rng.choice(n_features, n_informative,
                                 chunks=n_informative)
    beta = (rng.random(n_features, chunks=n_features) - 1) * scale

    informative_idx, beta = dask.compute(informative_idx, beta)

    z0 = X[:, informative_idx].dot(beta[informative_idx])
    y = rng.random(z0.shape, chunks=chunks[0]) < 1 / (1 + da.exp(-z0))
    y = y.astype(int)

    return X, y
Exemplo n.º 5
0
def black_scholes(nopt, price, strike, t, rate, vol, schd=None):
    mr = -rate
    sig_sig_two = vol * vol * 2

    P = price
    S = strike
    T = t

    a = log(P / S)
    b = T * mr

    z = T * sig_sig_two
    c = 0.25 * z
    y = da.map_blocks(invsqrt, z)

    w1 = (a - b + c) * y
    w2 = (a - b - c) * y

    d1 = 0.5 + 0.5 * da.map_blocks(erf, w1)
    d2 = 0.5 + 0.5 * da.map_blocks(erf, w2)

    Se = exp(b) * S

    call = P * d1 - Se * d2
    put = call - P + Se

    return da.compute(da.stack((put, call)), get=schd)
Exemplo n.º 6
0
def gaussian_kernel(x: Union[np.ndarray, da.array],
                    y: Union[np.ndarray, da.array],
                    sigma: np.ndarray = None) -> Union[np.ndarray, da.array]:
    """
    Gaussian kernel between samples of x and y. A sum of kernels is computed
    for multiple values of sigma.

    Parameters
    ----------
    x
        Batch of instances of shape [Nx, features].
    y
        Batch of instances of shape [Ny, features].
    sigma
        Array with values of the kernel width sigma.
    chunks
        Chunk sizes for x and y when using dask to compute the pairwise distances.

    Returns
    -------
    Array [Nx, Ny] of the kernel.
    """
    beta = 1. / (2. * np.expand_dims(sigma, 1))  # [Ns,1]
    dist = distance.pairwise_distance(x, y)  # [Nx,Ny]
    s = beta @ dist.reshape(1, -1)  # [Ns,1]*[1,Nx*Ny]=[Ns,Nx*Ny]
    s = np.exp(-s) if isinstance(s, np.ndarray) else da.exp(-s)
    return s.sum(axis=0).reshape(dist.shape)  # [Nx,Ny]
Exemplo n.º 7
0
def black_scholes_numpy_mod(nopt, price, strike, t, rate, vol):
    mr = -rate
    sig_sig_two = vol * vol * 2

    P = price
    S = strike
    T = t

    a = log(P / S)
    b = T * mr

    z = T * sig_sig_two
    c = 0.25 * z
    y = invsqrt(z)

    w1 = (a - b + c) * y
    w2 = (a - b - c) * y

    d1 = 0.5 + 0.5 * erf(w1)
    d2 = 0.5 + 0.5 * erf(w2)

    Se = exp(b) * S

    call = P * d1 - Se * d2
    put = call - P + Se

    return np.stack((call, put))
Exemplo n.º 8
0
def atm_variables_finder(mus,
                         muv,
                         phi,
                         height,
                         tau,
                         tO3,
                         tH2O,
                         taustep4sphalb,
                         tO2=1.0):
    tau_step = da.linspace(taustep4sphalb,
                           MAXNUMSPHALBVALUES * taustep4sphalb,
                           MAXNUMSPHALBVALUES,
                           chunks=int(MAXNUMSPHALBVALUES / 2))
    sphalb0 = csalbr(tau_step)
    taur = tau * da.exp(-height / SCALEHEIGHT)
    rhoray, trdown, trup = chand(phi, muv, mus, taur)
    if isinstance(height, xr.DataArray):

        def _sphalb_index(index_arr, sphalb0):
            # FIXME: if/when dask can support lazy index arrays then remove this
            return sphalb0[index_arr]

        sphalb = da.map_blocks(_sphalb_index, (taur / taustep4sphalb +
                                               0.5).astype(np.int32).data,
                               sphalb0.compute(),
                               dtype=sphalb0.dtype)
    else:
        sphalb = sphalb0[(taur / taustep4sphalb + 0.5).astype(np.int32)]
    Ttotrayu = ((2 / 3. + muv) + (2 / 3. - muv) * trup) / (4 / 3. + taur)
    Ttotrayd = ((2 / 3. + mus) + (2 / 3. - mus) * trdown) / (4 / 3. + taur)
    TtotraytH2O = Ttotrayu * Ttotrayd * tH2O
    tOG = tO3 * tO2
    return sphalb, rhoray, TtotraytH2O, tOG
def smooth(D, y, sigma):
    """Given a matrix ``D`` defining the squared distance between training and
    prediction points, and a matrix or vector y defining one or more lets of labels at
    the training points, return predicitons using an RBF kernel.

    Parameters
    ----------
    D : array-like
        Squared distance matrix. $D_{i,j}$ defines the squared distance between training
        point ``i`` and prediction point ``j``.
    y : array-like
        Labels for training points. May be 1d if a single prediction task or 2d if >1
        prediction tasks.
    sigma : int
        The length scale for RBF smoothing. i.e. the weight of a training point is
        proportional to $exp((-.5/sigma**2)*D_{i,j})$

    Returns
    -------
    smoothed_predictions : array-like
        Predicted labels
    """
    y = solve_functions.y_to_matrix(y)

    # get RBF smoothing matrix
    S = da.exp((-0.5 / sigma**2) * D)

    # if sigma is small enough, weights could turn to 0, so we reset those to
    # just guess the average of the training set
    S = da.where(S.sum(axis=1).reshape(-1, 1) > 0, S, 1)
    smoothed_predictions = S.dot(y) / S.sum(axis=1).reshape(-1, 1)

    return smoothed_predictions
Exemplo n.º 10
0
def logsumexp(arr, axis=0):
    """Computes the sum of arr assuming arr is in the log domain.
    Returns log(sum(exp(arr))) while minimizing the possibility of
    over/underflow.
    Examples
    --------
    >>> import numpy as np
    >>> from sklearn.utils.extmath import logsumexp
    >>> a = np.arange(10)
    >>> np.log(np.sum(np.exp(a)))
    9.4586297444267107
    >>> logsumexp(a)
    9.4586297444267107
    """
    if axis == 0:
        pass
    elif axis == 1:
        arr = arr.T
    else:
        raise NotImplementedError
    # Use the max to normalize, as with the log this is what accumulates
    # the less errors
    vmax = arr.max(axis=0)
    out = da.log(da.sum(da.exp(arr - vmax), axis=0))
    out += vmax
    return out
Exemplo n.º 11
0
def gradient(X, y, max_steps=100):
    N, M = X.shape
    firstBacktrackMult = 0.1
    nextBacktrackMult = 0.5
    armijoMult = 0.1
    stepGrowth = 1.25
    stepSize = 1.0
    recalcRate = 10
    backtrackMult = firstBacktrackMult
    beta = np.zeros(M)

    print('##       -f        |df/f|    |dx/x|    step')
    print('----------------------------------------------')
    for k in range(max_steps):
        # Compute the gradient
        if k % recalcRate == 0:
            Xbeta = X.dot(beta)
            eXbeta = da.exp(Xbeta)
            func = da.log1p(eXbeta).sum() - y.dot(Xbeta)
        e1 = eXbeta + 1.0
        gradient = X.T.dot(eXbeta / e1 - y)
        steplen = (gradient**2).sum()**0.5
        Xgradient = X.dot(gradient)

        Xbeta, eXbeta, func, gradient, steplen, Xgradient = da.compute(
            Xbeta, eXbeta, func, gradient, steplen, Xgradient)

        obeta = beta
        oXbeta = Xbeta

        # Compute the step size
        lf = func
        for ii in range(100):
            beta = obeta - stepSize * gradient
            if ii and np.array_equal(beta, obeta):
                stepSize = 0
                break
            Xbeta = oXbeta - stepSize * Xgradient
            # This prevents overflow
            if np.all(Xbeta < 700):
                eXbeta = np.exp(Xbeta)
                func = np.sum(np.log1p(eXbeta)) - np.dot(y, Xbeta)
                df = lf - func
                if df >= armijoMult * stepSize * steplen**2:
                    break
            stepSize *= backtrackMult
        if stepSize == 0:
            print('No more progress')
            break
        df /= max(func, lf)
        db = stepSize * steplen / (np.linalg.norm(beta) + stepSize * steplen)
        print('%2d  %.6e %9.2e  %.2e  %.1e' % (k + 1, func, df, db, stepSize))
        if df < 1e-14:
            print('Converged')
            break
        stepSize *= stepGrowth
        backtrackMult = nextBacktrackMult

    return beta
Exemplo n.º 12
0
def rbf_kernel(X, Y=None, gamma=None):
    X, Y = check_pairwise_arrays(X, Y)
    if gamma is None:
        gamma = 1.0 / X.shape[1]

    K = euclidean_distances(X, Y, squared=True)
    K = da.exp(-gamma * K)
    return K
Exemplo n.º 13
0
def make_classification(n_samples=1000, n_features=100, n_informative=2, scale=1.0,
                        chunksize=100):
    X = da.random.normal(0, 1, size=(n_samples, n_features),
                         chunks=(chunksize, n_features))
    informative_idx = np.random.choice(n_features, n_informative)
    beta = (np.random.random(n_features) - 1) * scale
    z0 = X[:, informative_idx].dot(beta[informative_idx])
    y = da.random.random(z0.shape, chunks=(chunksize,)) < 1 / (1 + da.exp(-z0))
    return X, y
Exemplo n.º 14
0
def get_atm_variables_abi(mus, muv, phi, height, G_O3, G_H2O, G_O2, ah2o, ao2,
                          ao3, tau):
    tO3 = 1.0
    tH2O = 1.0
    if ao3 != 0:
        tO3 = da.exp(-G_O3 * ao3)
    if ah2o != 0:
        tH2O = da.exp(-G_H2O * ah2o)
    tO2 = da.exp(-G_O2 * ao2)
    # Returns sphalb, rhoray, TtotraytH2O, tOG.
    return atm_variables_finder(mus,
                                muv,
                                phi,
                                height,
                                tau,
                                tO3,
                                tH2O,
                                TAUSTEP4SPHALB_ABI,
                                tO2=tO2)
Exemplo n.º 15
0
def rbf_kernel(X: ArrayLike,
               Y: Optional[ArrayLike] = None,
               gamma: Optional[float] = None) -> ArrayLike:
    X, Y = check_pairwise_arrays(X, Y)
    if gamma is None:
        gamma = 1.0 / X.shape[1]

    K = euclidean_distances(X, Y, squared=True)
    K = da.exp(-gamma * K)
    return K
Exemplo n.º 16
0
def make_counts(
    n_samples=1000,
    n_features=100,
    n_informative=2,
    scale=1.0,
    chunks=100,
    random_state=None,
):
    """
    Generate a dummy dataset for modeling count data.

    Parameters
    ----------
    n_samples : int
        number of rows in the output array
    n_features : int
        number of columns (features) in the output array
    n_informative : int
        number of features that are correlated with the outcome
    scale : float
        Scale the true coefficient array by this
    chunks : int
        Number of rows per dask array block.
    random_state : int, RandomState instance or None (default)
        Determines random number generation for dataset creation. Pass an int
        for reproducible output across multiple function calls.
        See :term:`Glossary <random_state>`.

    Returns
    -------
    X : dask.array, size ``(n_samples, n_features)``
    y : dask.array, size ``(n_samples,)``
        array of non-negative integer-valued data

    Examples
    --------
    >>> X, y = make_counts()
    """
    rng = dask_ml.utils.check_random_state(random_state)

    X = rng.normal(0,
                   1,
                   size=(n_samples, n_features),
                   chunks=(chunks, n_features))
    informative_idx = rng.choice(n_features,
                                 n_informative,
                                 chunks=n_informative)
    beta = (rng.random(n_features, chunks=n_features) - 1) * scale

    informative_idx, beta = dask.compute(informative_idx, beta)

    z0 = X[:, informative_idx].dot(beta[informative_idx])
    rate = da.exp(z0)
    y = rng.poisson(rate, size=1, chunks=(chunks, ))
    return X, y
Exemplo n.º 17
0
def gradient_descent(X, y, max_steps=100, tol=1e-14):
    '''Michael Grant's implementation of Gradient Descent.'''

    n, p = X.shape
    firstBacktrackMult = 0.1
    nextBacktrackMult = 0.5
    armijoMult = 0.1
    stepGrowth = 1.25
    stepSize = 1.0
    recalcRate = 10
    backtrackMult = firstBacktrackMult
    beta = np.zeros(p)
    y_local = y.compute()

    for k in range(max_steps):
        # how necessary is this recalculation?
        if k % recalcRate == 0:
            Xbeta = X.dot(beta)
            eXbeta = da.exp(Xbeta)
            func = da.log1p(eXbeta).sum() - y.dot(Xbeta)

        e1 = eXbeta + 1.0
        gradient = X.T.dot(eXbeta / e1 - y)
        Xgradient = X.dot(gradient)

        Xbeta, eXbeta, func, gradient, Xgradient = da.compute(
            Xbeta, eXbeta, func, gradient, Xgradient)

        # backtracking line search
        lf = func
        stepSize, beta, Xbeta, func = compute_stepsize(beta, gradient,
                                                       Xbeta, Xgradient,
                                                       y_local, func,
                                                       **{
                                                           'backtrackMult': backtrackMult,
                                                           'armijoMult': armijoMult,
                                                           'stepSize': stepSize})
        if stepSize == 0:
            print('No more progress')
            break

        # necessary for gradient computation
        eXbeta = exp(Xbeta)

        df = lf - func
        df /= max(func, lf)

        if df < tol:
            print('Converged')
            break
        stepSize *= stepGrowth
        backtrackMult = nextBacktrackMult

    return beta
Exemplo n.º 18
0
 def predict_proba(self, X):
     """
     Return probability estimates for the test vector X.
     Parameters
     ----------
     X : array-like, shape = [n_samples, n_features]
     Returns
     -------
     C : array-like, shape = [n_samples, n_classes]
         Returns the probability of the samples for each class in
         the model. The columns correspond to the classes in sorted
         order, as they appear in the attribute `classes_`.
     """
     return da.exp(self.predict_log_proba(X))
Exemplo n.º 19
0
def make_classification(n_samples=1000,
                        n_features=100,
                        n_informative=2,
                        scale=1.0,
                        chunksize=100,
                        is_sparse=False):
    """
    Generate a dummy dataset for classification tasks.

    Parameters
    ----------
    n_samples : int
        number of rows in the output array
    n_features : int
        number of columns (features) in the output array
    n_informative : int
        number of features that are correlated with the outcome
    scale : float
        Scale the true coefficient array by this
    chunksize : int
        Number of rows per dask array block.
    is_sparse: bool
        Return a sparse matrix

    Returns
    -------
    X : dask.array, size ``(n_samples, n_features)``
    y : dask.array, size ``(n_samples,)``
        boolean-valued array

    Examples
    --------
    >>> X, y = make_classification()
    >>> X
    dask.array<da.random.normal, shape=(1000, 100), dtype=float64, chunksize=(100, 100)>
    >>> y
    dask.array<lt, shape=(1000,), dtype=bool, chunksize=(100,)>
    """
    X = da.random.normal(0,
                         1,
                         size=(n_samples, n_features),
                         chunks=(chunksize, n_features))
    if is_sparse:
        X = X.map_blocks(sparse.COO)
    informative_idx = np.random.choice(n_features, n_informative)
    beta = (np.random.random(n_features) - 1) * scale
    z0 = X[:, informative_idx].dot(beta[informative_idx])
    y = da.random.random(z0.shape,
                         chunks=(chunksize, )) < 1 / (1 + da.exp(-z0))
    return X, y
Exemplo n.º 20
0
 def gram_rbf(X, threshold=1.0):
     if type(X) == torch.Tensor:
         dot_products = X @ X.t()
         sq_norms = dot_products.diag()
         sq_distances = -2*dot_products + sq_norms[:,None] + sq_norms[None,:]
         sq_median_distance = sq_distances.median()
         return torch.exp(-sq_distances / (2*threshold**2 * sq_median_distance))
     elif type(X) == da.Array:
         dot_products = X @ X.T
         sq_norms = da.diag(dot_products)
         sq_distances = -2*dot_products + sq_norms[:,None] + sq_norms[None,:]
         sq_median_distance = da.percentile(sq_distances.ravel(), 50)
         return da.exp((-sq_distances / (2*threshold**2 * sq_median_distance)))
     else:
         raise ValueError
Exemplo n.º 21
0
def test_dtype_complex():
    x = np.arange(24).reshape((4, 6)).astype('f4')
    y = np.arange(24).reshape((4, 6)).astype('i8')
    z = np.arange(24).reshape((4, 6)).astype('i2')

    a = da.from_array(x, chunks=(2, 3))
    b = da.from_array(y, chunks=(2, 3))
    c = da.from_array(z, chunks=(2, 3))

    def eq(a, b):
        return (isinstance(a, np.dtype) and
                isinstance(b, np.dtype) and
                str(a) == str(b))

    assert eq(a._dtype, x.dtype)
    assert eq(b._dtype, y.dtype)

    assert eq((a + 1)._dtype, (x + 1).dtype)
    assert eq((a + b)._dtype, (x + y).dtype)
    assert eq(a.T._dtype, x.T.dtype)
    assert eq(a[:3]._dtype, x[:3].dtype)
    assert eq((a.dot(b.T))._dtype, (x.dot(y.T)).dtype)

    assert eq(stack([a, b])._dtype, np.vstack([x, y]).dtype)
    assert eq(concatenate([a, b])._dtype, np.concatenate([x, y]).dtype)

    assert eq(b.std()._dtype, y.std().dtype)
    assert eq(c.sum()._dtype, z.sum().dtype)
    assert eq(a.min()._dtype, a.min().dtype)
    assert eq(b.std()._dtype, b.std().dtype)
    assert eq(a.argmin(axis=0)._dtype, a.argmin(axis=0).dtype)

    assert eq(da.sin(c)._dtype, np.sin(z).dtype)
    assert eq(da.exp(b)._dtype, np.exp(y).dtype)
    assert eq(da.floor(a)._dtype, np.floor(x).dtype)
    assert eq(da.isnan(b)._dtype, np.isnan(y).dtype)
    with ignoring(ImportError):
        assert da.isnull(b)._dtype == 'bool'
        assert da.notnull(b)._dtype == 'bool'

    x = np.array([('a', 1)], dtype=[('text', 'S1'), ('numbers', 'i4')])
    d = da.from_array(x, chunks=(1,))

    assert eq(d['text']._dtype, x['text'].dtype)
    assert eq(d[['numbers', 'text']]._dtype, x[['numbers', 'text']].dtype)
Exemplo n.º 22
0
def test_dtype_complex():
    x = np.arange(24).reshape((4, 6)).astype('f4')
    y = np.arange(24).reshape((4, 6)).astype('i8')
    z = np.arange(24).reshape((4, 6)).astype('i2')

    a = da.from_array(x, chunks=(2, 3))
    b = da.from_array(y, chunks=(2, 3))
    c = da.from_array(z, chunks=(2, 3))

    def eq(a, b):
        return (isinstance(a, np.dtype) and isinstance(b, np.dtype)
                and str(a) == str(b))

    assert eq(a._dtype, x.dtype)
    assert eq(b._dtype, y.dtype)

    assert eq((a + 1)._dtype, (x + 1).dtype)
    assert eq((a + b)._dtype, (x + y).dtype)
    assert eq(a.T._dtype, x.T.dtype)
    assert eq(a[:3]._dtype, x[:3].dtype)
    assert eq((a.dot(b.T))._dtype, (x.dot(y.T)).dtype)

    assert eq(stack([a, b])._dtype, np.vstack([x, y]).dtype)
    assert eq(concatenate([a, b])._dtype, np.concatenate([x, y]).dtype)

    assert eq(b.std()._dtype, y.std().dtype)
    assert eq(c.sum()._dtype, z.sum().dtype)
    assert eq(a.min()._dtype, a.min().dtype)
    assert eq(b.std()._dtype, b.std().dtype)
    assert eq(a.argmin(axis=0)._dtype, a.argmin(axis=0).dtype)

    assert eq(da.sin(c)._dtype, np.sin(z).dtype)
    assert eq(da.exp(b)._dtype, np.exp(y).dtype)
    assert eq(da.floor(a)._dtype, np.floor(x).dtype)
    assert eq(da.isnan(b)._dtype, np.isnan(y).dtype)
    with ignoring(ImportError):
        assert da.isnull(b)._dtype == 'bool'
        assert da.notnull(b)._dtype == 'bool'

    x = np.array([('a', 1)], dtype=[('text', 'S1'), ('numbers', 'i4')])
    d = da.from_array(x, chunks=(1, ))

    assert eq(d['text']._dtype, x['text'].dtype)
    assert eq(d[['numbers', 'text']]._dtype, x[['numbers', 'text']].dtype)
Exemplo n.º 23
0
def matern32(coords, lambda0):
    """ Matern 3/2 covariance kernel.

    Parameters
    ----------
    coords: (n_pts, n_dims) dask.array or Future
        Point coordinates.

    Returns
    -------
    covs: (n_pts, n_pts) delayed dask.array
        Pairwise covariance matrix.

    """
    dists = dask_distance.euclidean(coords)
    res = da.multiply(
            1 + (np.sqrt(3) / lambda0) * dists,
            da.exp(-(np.sqrt(3) / lambda0) * dists))
    return res
Exemplo n.º 24
0
def make_counts(n_samples=1000,
                n_features=100,
                n_informative=2,
                scale=1.0,
                chunks=100):
    """
    Generate a dummy dataset for modeling count data.

    Parameters
    ----------
    n_samples : int
        number of rows in the output array
    n_features : int
        number of columns (features) in the output array
    n_informative : int
        number of features that are correlated with the outcome
    scale : float
        Scale the true coefficient array by this
    chunks : int
        Number of rows per dask array block.

    Returns
    -------
    X : dask.array, size ``(n_samples, n_features)``
    y : dask.array, size ``(n_samples,)``
        array of non-negative integer-valued data

    Examples
    --------
    >>> X, y = make_counts()
    """
    X = da.random.normal(0,
                         1,
                         size=(n_samples, n_features),
                         chunks=(chunks, n_features))
    informative_idx = np.random.choice(n_features, n_informative)
    beta = (np.random.random(n_features) - 1) * scale
    z0 = X[:, informative_idx].dot(beta[informative_idx])
    rate = da.exp(z0)
    y = da.random.poisson(rate, size=1, chunks=(chunks, ))
    return X, y
Exemplo n.º 25
0
    def get_harmonics(self, in_sphere):
        """Create the harmonics for this arrangement of sphere pixels"""
        # cache_key = "{}:".format(in_sphere.npix)
        # if (cache_key in self.harmonics):
        # return self.harmonics[cache_key]

        harmonic_list = []
        p2j = jomega(self.frequency)

        # logger.info("pixel areas:  {}".format(in_sphere.pixel_areas))
        for u, v, w in zip(self.u_arr, self.v_arr, self.w_arr):
            harmonic = (da.exp(
                p2j *
                (u * in_sphere.l + v * in_sphere.m + w * in_sphere.n_minus_1))
                        * in_sphere.pixel_areas)
            assert harmonic.shape[0] == in_sphere.npix
            harmonic_list.append(harmonic)
        # self.harmonics[cache_key] = harmonic_list

        # assert(harmonic_list[0].shape[0] == in_sphere.npix)
        return harmonic_list
Exemplo n.º 26
0
def fourier_shift(image, shift, n=-1, axis=-1):
    """
    Multi-dimensional fourier shift filter.

    The array is multiplied with the fourier transform of a shift operation.

    Parameters
    ----------
    image : array_like
        The input image.
    shift : float or sequence
        The size of the box used for filtering.
        If a float, `shift` is the same for all axes. If a sequence, `shift`
        has to contain one value for each axis.
    n : int, optional
        If `n` is negative (default), then the image is assumed to be the
        result of a complex fft.
        If `n` is larger than or equal to zero, the image is assumed to be the
        result of a real fft, and `n` gives the length of the array before
        transformation along the real transform direction.
    axis : int, optional
        The axis of the real transform.

    Returns
    -------
    fourier_shift : Dask Array

    Examples
    --------
    >>> from scipy import ndimage, misc
    >>> import matplotlib.pyplot as plt
    >>> import numpy.fft
    >>> fig, (ax1, ax2) = plt.subplots(1, 2)
    >>> plt.gray()  # show the filtered result in grayscale
    >>> ascent = misc.ascent()
    >>> image = numpy.fft.fft2(ascent)
    >>> result = ndimage.fourier_shift(image, shift=200)
    >>> result = numpy.fft.ifft2(result)
    >>> ax1.imshow(ascent)
    >>> ax2.imshow(result.real)  # the imaginary part is an artifact
    >>> plt.show()
    """

    if issubclass(image.dtype.type, numbers.Real):
        image = image.astype(complex)

    # Validate and normalize arguments
    image, shift, n, axis = _utils._norm_args(image, shift, n=n, axis=axis)

    # Constants with type converted
    J = image.dtype.type(1j)

    # Get the grid of frequencies
    ang_freq_grid = _utils._get_ang_freq_grid(image.shape,
                                              chunks=image.chunks,
                                              n=n,
                                              axis=axis,
                                              dtype=shift.dtype)

    # Apply shift
    result = image.copy()
    for ax, f in enumerate(ang_freq_grid):
        phase_shift = da.exp((-J) * shift[ax] * f)
        phase_shift = _utils._reshape_nd(phase_shift, ndim=image.ndim, axis=ax)
        result *= phase_shift

    return result
Exemplo n.º 27
0
def test_arithmetic():
    x = np.arange(5).astype('f4') + 2
    y = np.arange(5).astype('i8') + 2
    z = np.arange(5).astype('i4') + 2
    a = da.from_array(x, chunks=(2,))
    b = da.from_array(y, chunks=(2,))
    c = da.from_array(z, chunks=(2,))
    assert eq(a + b, x + y)
    assert eq(a * b, x * y)
    assert eq(a - b, x - y)
    assert eq(a / b, x / y)
    assert eq(b & b, y & y)
    assert eq(b | b, y | y)
    assert eq(b ^ b, y ^ y)
    assert eq(a // b, x // y)
    assert eq(a ** b, x ** y)
    assert eq(a % b, x % y)
    assert eq(a > b, x > y)
    assert eq(a < b, x < y)
    assert eq(a >= b, x >= y)
    assert eq(a <= b, x <= y)
    assert eq(a == b, x == y)
    assert eq(a != b, x != y)

    assert eq(a + 2, x + 2)
    assert eq(a * 2, x * 2)
    assert eq(a - 2, x - 2)
    assert eq(a / 2, x / 2)
    assert eq(b & True, y & True)
    assert eq(b | True, y | True)
    assert eq(b ^ True, y ^ True)
    assert eq(a // 2, x // 2)
    assert eq(a ** 2, x ** 2)
    assert eq(a % 2, x % 2)
    assert eq(a > 2, x > 2)
    assert eq(a < 2, x < 2)
    assert eq(a >= 2, x >= 2)
    assert eq(a <= 2, x <= 2)
    assert eq(a == 2, x == 2)
    assert eq(a != 2, x != 2)

    assert eq(2 + b, 2 + y)
    assert eq(2 * b, 2 * y)
    assert eq(2 - b, 2 - y)
    assert eq(2 / b, 2 / y)
    assert eq(True & b, True & y)
    assert eq(True | b, True | y)
    assert eq(True ^ b, True ^ y)
    assert eq(2 // b, 2 // y)
    assert eq(2 ** b, 2 ** y)
    assert eq(2 % b, 2 % y)
    assert eq(2 > b, 2 > y)
    assert eq(2 < b, 2 < y)
    assert eq(2 >= b, 2 >= y)
    assert eq(2 <= b, 2 <= y)
    assert eq(2 == b, 2 == y)
    assert eq(2 != b, 2 != y)

    assert eq(-a, -x)
    assert eq(abs(a), abs(x))
    assert eq(~(a == b), ~(x == y))
    assert eq(~(a == b), ~(x == y))

    assert eq(da.logaddexp(a, b), np.logaddexp(x, y))
    assert eq(da.logaddexp2(a, b), np.logaddexp2(x, y))
    assert eq(da.exp(b), np.exp(y))
    assert eq(da.log(a), np.log(x))
    assert eq(da.log10(a), np.log10(x))
    assert eq(da.log1p(a), np.log1p(x))
    assert eq(da.expm1(b), np.expm1(y))
    assert eq(da.sqrt(a), np.sqrt(x))
    assert eq(da.square(a), np.square(x))

    assert eq(da.sin(a), np.sin(x))
    assert eq(da.cos(b), np.cos(y))
    assert eq(da.tan(a), np.tan(x))
    assert eq(da.arcsin(b/10), np.arcsin(y/10))
    assert eq(da.arccos(b/10), np.arccos(y/10))
    assert eq(da.arctan(b/10), np.arctan(y/10))
    assert eq(da.arctan2(b*10, a), np.arctan2(y*10, x))
    assert eq(da.hypot(b, a), np.hypot(y, x))
    assert eq(da.sinh(a), np.sinh(x))
    assert eq(da.cosh(b), np.cosh(y))
    assert eq(da.tanh(a), np.tanh(x))
    assert eq(da.arcsinh(b*10), np.arcsinh(y*10))
    assert eq(da.arccosh(b*10), np.arccosh(y*10))
    assert eq(da.arctanh(b/10), np.arctanh(y/10))
    assert eq(da.deg2rad(a), np.deg2rad(x))
    assert eq(da.rad2deg(a), np.rad2deg(x))

    assert eq(da.logical_and(a < 1, b < 4), np.logical_and(x < 1, y < 4))
    assert eq(da.logical_or(a < 1, b < 4), np.logical_or(x < 1, y < 4))
    assert eq(da.logical_xor(a < 1, b < 4), np.logical_xor(x < 1, y < 4))
    assert eq(da.logical_not(a < 1), np.logical_not(x < 1))
    assert eq(da.maximum(a, 5 - a), np.maximum(a, 5 - a))
    assert eq(da.minimum(a, 5 - a), np.minimum(a, 5 - a))
    assert eq(da.fmax(a, 5 - a), np.fmax(a, 5 - a))
    assert eq(da.fmin(a, 5 - a), np.fmin(a, 5 - a))

    assert eq(da.isreal(a + 1j * b), np.isreal(x + 1j * y))
    assert eq(da.iscomplex(a + 1j * b), np.iscomplex(x + 1j * y))
    assert eq(da.isfinite(a), np.isfinite(x))
    assert eq(da.isinf(a), np.isinf(x))
    assert eq(da.isnan(a), np.isnan(x))
    assert eq(da.signbit(a - 3), np.signbit(x - 3))
    assert eq(da.copysign(a - 3, b), np.copysign(x - 3, y))
    assert eq(da.nextafter(a - 3, b), np.nextafter(x - 3, y))
    assert eq(da.ldexp(c, c), np.ldexp(z, z))
    assert eq(da.fmod(a * 12, b), np.fmod(x * 12, y))
    assert eq(da.floor(a * 0.5), np.floor(x * 0.5))
    assert eq(da.ceil(a), np.ceil(x))
    assert eq(da.trunc(a / 2), np.trunc(x / 2))

    assert eq(da.degrees(b), np.degrees(y))
    assert eq(da.radians(a), np.radians(x))

    assert eq(da.rint(a + 0.3), np.rint(x + 0.3))
    assert eq(da.fix(a - 2.5), np.fix(x - 2.5))

    assert eq(da.angle(a + 1j), np.angle(x + 1j))
    assert eq(da.real(a + 1j), np.real(x + 1j))
    assert eq((a + 1j).real, np.real(x + 1j))
    assert eq(da.imag(a + 1j), np.imag(x + 1j))
    assert eq((a + 1j).imag, np.imag(x + 1j))
    assert eq(da.conj(a + 1j * b), np.conj(x + 1j * y))
    assert eq((a + 1j * b).conj(), (x + 1j * y).conj())

    assert eq(da.clip(b, 1, 4), np.clip(y, 1, 4))
    assert eq(da.fabs(b), np.fabs(y))
    assert eq(da.sign(b - 2), np.sign(y - 2))

    l1, l2 = da.frexp(a)
    r1, r2 = np.frexp(x)
    assert eq(l1, r1)
    assert eq(l2, r2)

    l1, l2 = da.modf(a)
    r1, r2 = np.modf(x)
    assert eq(l1, r1)
    assert eq(l2, r2)

    assert eq(da.around(a, -1), np.around(x, -1))
def softmax_dask(x):
    x = da.from_array(x, chunks=int(x.shape[0] / CPU_COUNT), name='x')
    e_x = da.exp(x - x.max())
    out = e_x / e_x.sum()
    normalized = out.compute()
    return normalized
Exemplo n.º 29
0
def pressure(x, y, t, t0=10000., U=50., a=200000., p0=2000., x0=0.):
    return (p0 * (1. - da.exp(-t / t0)) * da.exp(-((x - x0)**2. +
                                                   (y - U * t)**2.) / a**2.))
Exemplo n.º 30
0
 def _set_amplitude(self, amplitude):
     if isinstance(amplitude, BaseSignal):
         amplitude = amplitude.data.real
     self.data = amplitude * da.exp(1j * da.angle(self.data))
     self.events.data_changed.trigger(self)
Exemplo n.º 31
0
 def _set_phase(self, phase):
     if isinstance(phase, BaseSignal):
         phase = phase.data.real
     self.data = abs(self.data) * \
         da.exp(1j * phase)
     self.events.data_changed.trigger(self)
Exemplo n.º 32
0
 def _set_amplitude(self, amplitude):
     if isinstance(amplitude, BaseSignal):
         amplitude = amplitude.data.real
     self.data = amplitude * da.exp(1j * da.angle(self.data))
     self.events.data_changed.trigger(self)
Exemplo n.º 33
0
 def _set_phase(self, phase):
     if isinstance(phase, BaseSignal):
         phase = phase.data.real
     self.data = da.numpy_compat.builtins.abs(self.data) * \
         da.exp(1j * phase)
     self.events.data_changed.trigger(self)
Exemplo n.º 34
0
def exp(A):
    return da.exp(A)
Exemplo n.º 35
0
    def make_dirty(self):
        print("Making dirty", file=log)
        dirty = da.zeros((self.nband, self.nx, self.ny),
                         dtype=np.float32,
                         chunks=(1, self.nx, self.ny),
                         name=False)
        dirties = []
        for ims in self.ms:
            xds = xds_from_ms(ims,
                              group_cols=('FIELD_ID', 'DATA_DESC_ID'),
                              chunks=self.chunks[ims],
                              columns=self.columns)

            # subtables
            ddids = xds_from_table(ims + "::DATA_DESCRIPTION")
            fields = xds_from_table(ims + "::FIELD", group_cols="__row__")
            spws = xds_from_table(ims + "::SPECTRAL_WINDOW",
                                  group_cols="__row__")
            pols = xds_from_table(ims + "::POLARIZATION", group_cols="__row__")

            # subtable data
            ddids = dask.compute(ddids)[0]
            fields = dask.compute(fields)[0]
            spws = dask.compute(spws)[0]
            pols = dask.compute(pols)[0]

            for ds in xds:
                field = fields[ds.FIELD_ID]
                radec = field.PHASE_DIR.data.squeeze()
                if not np.array_equal(radec, self.radec):
                    continue

                spw = ds.DATA_DESC_ID  # this is not correct, need to use spw

                freq_bin_idx = self.freq_bin_idx[ims][spw]
                freq_bin_counts = self.freq_bin_counts[ims][spw]
                freq = self.freq[ims][spw]
                freq_chunk = freq_bin_counts[0].compute()

                uvw = ds.UVW.data

                data = getattr(ds, self.data_column).data
                dataxx = data[:, :, 0]
                datayy = data[:, :, -1]

                weights = getattr(ds, self.weight_column).data
                if len(weights.shape) < 3:
                    weights = da.broadcast_to(weights[:, None, :],
                                              data.shape,
                                              chunks=data.chunks)

                if self.imaging_weight_column is not None:
                    imaging_weights = getattr(ds,
                                              self.imaging_weight_column).data
                    if len(imaging_weights.shape) < 3:
                        imaging_weights = da.broadcast_to(
                            imaging_weights[:, None, :],
                            data.shape,
                            chunks=data.chunks)

                    weightsxx = imaging_weights[:, :, 0] * weights[:, :, 0]
                    weightsyy = imaging_weights[:, :, -1] * weights[:, :, -1]
                else:
                    weightsxx = weights[:, :, 0]
                    weightsyy = weights[:, :, -1]

                # apply adjoint of mueller term.
                # Phases modify data amplitudes modify weights.
                if self.mueller_column is not None:
                    mueller = getattr(ds, self.mueller_column).data
                    dataxx *= da.exp(-1j * da.angle(mueller[:, :, 0]))
                    datayy *= da.exp(-1j * da.angle(mueller[:, :, -1]))
                    weightsxx *= da.absolute(mueller[:, :, 0])
                    weightsyy *= da.absolute(mueller[:, :, -1])

                # weighted sum corr to Stokes I
                weights = weightsxx + weightsyy
                data = (weightsxx * dataxx + weightsyy * datayy)
                # TODO - turn off this stupid warning
                data = da.where(weights, data / weights, 0.0j)

                # only keep data where both corrs are unflagged
                flag = getattr(ds, self.flag_column).data
                flagxx = flag[:, :, 0]
                flagyy = flag[:, :, -1]
                # ducc0 convention uses uint8 mask not flag
                flag = ~(flagxx | flagyy)

                dirty = vis2im(uvw,
                               freq,
                               data,
                               freq_bin_idx,
                               freq_bin_counts,
                               self.nx,
                               self.ny,
                               self.cell,
                               weights=weights,
                               flag=flag.astype(np.uint8),
                               nthreads=self.nthreads,
                               epsilon=self.epsilon,
                               do_wstacking=self.do_wstacking,
                               double_accum=True)

                dirties.append(dirty)

        dirties = dask.compute(dirties, scheduler='single-threaded')[0]

        return accumulate_dirty(dirties, self.nband,
                                self.band_mapping).astype(self.real_type)
Exemplo n.º 36
0
def chand(phi, muv, mus, taur):
    # FROM FUNCTION CHAND
    # phi: azimuthal difference between sun and observation in degree
    #      (phi=0 in backscattering direction)
    # mus: cosine of the sun zenith angle
    # muv: cosine of the observation zenith angle
    # taur: molecular optical depth
    # rhoray: molecular path reflectance
    # constant xdep: depolarization factor (0.0279)
    #          xfd = (1-xdep/(2-xdep)) / (1 + 2*xdep/(2-xdep)) = 2 * (1 - xdep) / (2 + xdep) = 0.958725775
    # */
    xfd = 0.958725775
    xbeta2 = 0.5
    #         float pl[5];
    #         double fs01, fs02, fs0, fs1, fs2;
    as0 = [
        0.33243832, 0.16285370, -0.30924818, -0.10324388, 0.11493334,
        -6.777104e-02, 1.577425e-03, -1.240906e-02, 3.241678e-02, -3.503695e-02
    ]
    as1 = [0.19666292, -5.439061e-02]
    as2 = [0.14545937, -2.910845e-02]
    #         float phios, xcos1, xcos2, xcos3;
    #         float xph1, xph2, xph3, xitm1, xitm2;
    #         float xlntaur, xitot1, xitot2, xitot3;
    #         int i, ib;

    xph1 = 1.0 + (3.0 * mus * mus - 1.0) * (3.0 * muv * muv - 1.0) * xfd / 8.0
    xph2 = -xfd * xbeta2 * 1.5 * mus * muv * da.sqrt(
        1.0 - mus * mus) * da.sqrt(1.0 - muv * muv)
    xph3 = xfd * xbeta2 * 0.375 * (1.0 - mus * mus) * (1.0 - muv * muv)

    # pl[0] = 1.0
    # pl[1] = mus + muv
    # pl[2] = mus * muv
    # pl[3] = mus * mus + muv * muv
    # pl[4] = mus * mus * muv * muv

    fs01 = as0[0] + (mus + muv) * as0[1] + (mus * muv) * as0[2] + (
        mus * mus + muv * muv) * as0[3] + (mus * mus * muv * muv) * as0[4]
    fs02 = as0[5] + (mus + muv) * as0[6] + (mus * muv) * as0[7] + (
        mus * mus + muv * muv) * as0[8] + (mus * mus * muv * muv) * as0[9]
    #         for (i = 0; i < 5; i++) {
    #                 fs01 += (double) (pl[i] * as0[i]);
    #                 fs02 += (double) (pl[i] * as0[5 + i]);
    #         }

    # for refl, (ah2o, bh2o, ao3, tau) in zip(reflectance_bands, coefficients):

    # ib = find_coefficient_index(center_wl)
    # if ib is None:
    #     raise ValueError("Can't handle band with wavelength '{}'".format(center_wl))

    xlntaur = da.log(taur)

    fs0 = fs01 + fs02 * xlntaur
    fs1 = as1[0] + xlntaur * as1[1]
    fs2 = as2[0] + xlntaur * as2[1]
    del xlntaur, fs01, fs02

    trdown = da.exp(-taur / mus)
    trup = da.exp(-taur / muv)

    xitm1 = (1.0 - trdown * trup) / 4.0 / (mus + muv)
    xitm2 = (1.0 - trdown) * (1.0 - trup)
    xitot1 = xph1 * (xitm1 + xitm2 * fs0)
    xitot2 = xph2 * (xitm1 + xitm2 * fs1)
    xitot3 = xph3 * (xitm1 + xitm2 * fs2)
    del xph1, xph2, xph3, xitm1, xitm2, fs0, fs1, fs2

    phios = da.deg2rad(phi + 180.0)
    xcos1 = 1.0
    xcos2 = da.cos(phios)
    xcos3 = da.cos(2.0 * phios)
    del phios

    rhoray = xitot1 * xcos1 + xitot2 * xcos2 * 2.0 + xitot3 * xcos3 * 2.0
    return rhoray, trdown, trup