def test_arithmetic():
x = np.arange(5).astype('f4') + 2
y = np.arange(5).astype('i8') + 2
z = np.arange(5).astype('i4') + 2
a = da.from_array(x, chunks=(2,))
b = da.from_array(y, chunks=(2,))
c = da.from_array(z, chunks=(2,))
assert eq(a + b, x + y)
assert eq(a * b, x * y)
assert eq(a - b, x - y)
assert eq(a / b, x / y)
assert eq(b & b, y & y)
assert eq(b | b, y | y)
assert eq(b ^ b, y ^ y)
assert eq(a // b, x // y)
assert eq(a ** b, x ** y)
assert eq(a % b, x % y)
assert eq(a > b, x > y)
assert eq(a < b, x < y)
assert eq(a >= b, x >= y)
assert eq(a <= b, x <= y)
assert eq(a == b, x == y)
assert eq(a != b, x != y)
assert eq(a + 2, x + 2)
assert eq(a * 2, x * 2)
assert eq(a - 2, x - 2)
assert eq(a / 2, x / 2)
assert eq(b & True, y & True)
assert eq(b | True, y | True)
assert eq(b ^ True, y ^ True)
assert eq(a // 2, x // 2)
assert eq(a ** 2, x ** 2)
assert eq(a % 2, x % 2)
assert eq(a > 2, x > 2)
assert eq(a < 2, x < 2)
assert eq(a >= 2, x >= 2)
assert eq(a <= 2, x <= 2)
assert eq(a == 2, x == 2)
assert eq(a != 2, x != 2)
assert eq(2 + b, 2 + y)
assert eq(2 * b, 2 * y)
assert eq(2 - b, 2 - y)
assert eq(2 / b, 2 / y)
assert eq(True & b, True & y)
assert eq(True | b, True | y)
assert eq(True ^ b, True ^ y)
assert eq(2 // b, 2 // y)
assert eq(2 ** b, 2 ** y)
assert eq(2 % b, 2 % y)
assert eq(2 > b, 2 > y)
assert eq(2 < b, 2 < y)
assert eq(2 >= b, 2 >= y)
assert eq(2 <= b, 2 <= y)
assert eq(2 == b, 2 == y)
assert eq(2 != b, 2 != y)
assert eq(-a, -x)
assert eq(abs(a), abs(x))
assert eq(~(a == b), ~(x == y))
assert eq(~(a == b), ~(x == y))
assert eq(da.logaddexp(a, b), np.logaddexp(x, y))
assert eq(da.logaddexp2(a, b), np.logaddexp2(x, y))
assert eq(da.exp(b), np.exp(y))
assert eq(da.log(a), np.log(x))
assert eq(da.log10(a), np.log10(x))
assert eq(da.log1p(a), np.log1p(x))
assert eq(da.expm1(b), np.expm1(y))
assert eq(da.sqrt(a), np.sqrt(x))
assert eq(da.square(a), np.square(x))
assert eq(da.sin(a), np.sin(x))
assert eq(da.cos(b), np.cos(y))
assert eq(da.tan(a), np.tan(x))
assert eq(da.arcsin(b/10), np.arcsin(y/10))
assert eq(da.arccos(b/10), np.arccos(y/10))
assert eq(da.arctan(b/10), np.arctan(y/10))
assert eq(da.arctan2(b*10, a), np.arctan2(y*10, x))
assert eq(da.hypot(b, a), np.hypot(y, x))
assert eq(da.sinh(a), np.sinh(x))
assert eq(da.cosh(b), np.cosh(y))
assert eq(da.tanh(a), np.tanh(x))
assert eq(da.arcsinh(b*10), np.arcsinh(y*10))
assert eq(da.arccosh(b*10), np.arccosh(y*10))
assert eq(da.arctanh(b/10), np.arctanh(y/10))
assert eq(da.deg2rad(a), np.deg2rad(x))
assert eq(da.rad2deg(a), np.rad2deg(x))
assert eq(da.logical_and(a < 1, b < 4), np.logical_and(x < 1, y < 4))
assert eq(da.logical_or(a < 1, b < 4), np.logical_or(x < 1, y < 4))
assert eq(da.logical_xor(a < 1, b < 4), np.logical_xor(x < 1, y < 4))
assert eq(da.logical_not(a < 1), np.logical_not(x < 1))
assert eq(da.maximum(a, 5 - a), np.maximum(a, 5 - a))
assert eq(da.minimum(a, 5 - a), np.minimum(a, 5 - a))
assert eq(da.fmax(a, 5 - a), np.fmax(a, 5 - a))
assert eq(da.fmin(a, 5 - a), np.fmin(a, 5 - a))
assert eq(da.isreal(a + 1j * b), np.isreal(x + 1j * y))
assert eq(da.iscomplex(a + 1j * b), np.iscomplex(x + 1j * y))
assert eq(da.isfinite(a), np.isfinite(x))
assert eq(da.isinf(a), np.isinf(x))
assert eq(da.isnan(a), np.isnan(x))
assert eq(da.signbit(a - 3), np.signbit(x - 3))
assert eq(da.copysign(a - 3, b), np.copysign(x - 3, y))
assert eq(da.nextafter(a - 3, b), np.nextafter(x - 3, y))
assert eq(da.ldexp(c, c), np.ldexp(z, z))
assert eq(da.fmod(a * 12, b), np.fmod(x * 12, y))
assert eq(da.floor(a * 0.5), np.floor(x * 0.5))
assert eq(da.ceil(a), np.ceil(x))
assert eq(da.trunc(a / 2), np.trunc(x / 2))
assert eq(da.degrees(b), np.degrees(y))
assert eq(da.radians(a), np.radians(x))
assert eq(da.rint(a + 0.3), np.rint(x + 0.3))
assert eq(da.fix(a - 2.5), np.fix(x - 2.5))
assert eq(da.angle(a + 1j), np.angle(x + 1j))
assert eq(da.real(a + 1j), np.real(x + 1j))
assert eq((a + 1j).real, np.real(x + 1j))
assert eq(da.imag(a + 1j), np.imag(x + 1j))
assert eq((a + 1j).imag, np.imag(x + 1j))
assert eq(da.conj(a + 1j * b), np.conj(x + 1j * y))
assert eq((a + 1j * b).conj(), (x + 1j * y).conj())
assert eq(da.clip(b, 1, 4), np.clip(y, 1, 4))
assert eq(da.fabs(b), np.fabs(y))
assert eq(da.sign(b - 2), np.sign(y - 2))
l1, l2 = da.frexp(a)
r1, r2 = np.frexp(x)
assert eq(l1, r1)
assert eq(l2, r2)
l1, l2 = da.modf(a)
r1, r2 = np.modf(x)
assert eq(l1, r1)
assert eq(l2, r2)
assert eq(da.around(a, -1), np.around(x, -1))
def test_arithmetic():
x = np.arange(5).astype('f4') + 2
y = np.arange(5).astype('i8') + 2
z = np.arange(5).astype('i4') + 2
a = da.from_array(x, chunks=(2, ))
b = da.from_array(y, chunks=(2, ))
c = da.from_array(z, chunks=(2, ))
assert eq(a + b, x + y)
assert eq(a * b, x * y)
assert eq(a - b, x - y)
assert eq(a / b, x / y)
assert eq(b & b, y & y)
assert eq(b | b, y | y)
assert eq(b ^ b, y ^ y)
assert eq(a // b, x // y)
assert eq(a**b, x**y)
assert eq(a % b, x % y)
assert eq(a > b, x > y)
assert eq(a < b, x < y)
assert eq(a >= b, x >= y)
assert eq(a <= b, x <= y)
assert eq(a == b, x == y)
assert eq(a != b, x != y)
assert eq(a + 2, x + 2)
assert eq(a * 2, x * 2)
assert eq(a - 2, x - 2)
assert eq(a / 2, x / 2)
assert eq(b & True, y & True)
assert eq(b | True, y | True)
assert eq(b ^ True, y ^ True)
assert eq(a // 2, x // 2)
assert eq(a**2, x**2)
assert eq(a % 2, x % 2)
assert eq(a > 2, x > 2)
assert eq(a < 2, x < 2)
assert eq(a >= 2, x >= 2)
assert eq(a <= 2, x <= 2)
assert eq(a == 2, x == 2)
assert eq(a != 2, x != 2)
assert eq(2 + b, 2 + y)
assert eq(2 * b, 2 * y)
assert eq(2 - b, 2 - y)
assert eq(2 / b, 2 / y)
assert eq(True & b, True & y)
assert eq(True | b, True | y)
assert eq(True ^ b, True ^ y)
assert eq(2 // b, 2 // y)
assert eq(2**b, 2**y)
assert eq(2 % b, 2 % y)
assert eq(2 > b, 2 > y)
assert eq(2 < b, 2 < y)
assert eq(2 >= b, 2 >= y)
assert eq(2 <= b, 2 <= y)
assert eq(2 == b, 2 == y)
assert eq(2 != b, 2 != y)
assert eq(-a, -x)
assert eq(abs(a), abs(x))
assert eq(~(a == b), ~(x == y))
assert eq(~(a == b), ~(x == y))
assert eq(da.logaddexp(a, b), np.logaddexp(x, y))
assert eq(da.logaddexp2(a, b), np.logaddexp2(x, y))
assert eq(da.exp(b), np.exp(y))
assert eq(da.log(a), np.log(x))
assert eq(da.log10(a), np.log10(x))
assert eq(da.log1p(a), np.log1p(x))
assert eq(da.expm1(b), np.expm1(y))
assert eq(da.sqrt(a), np.sqrt(x))
assert eq(da.square(a), np.square(x))
assert eq(da.sin(a), np.sin(x))
assert eq(da.cos(b), np.cos(y))
assert eq(da.tan(a), np.tan(x))
assert eq(da.arcsin(b / 10), np.arcsin(y / 10))
assert eq(da.arccos(b / 10), np.arccos(y / 10))
assert eq(da.arctan(b / 10), np.arctan(y / 10))
assert eq(da.arctan2(b * 10, a), np.arctan2(y * 10, x))
assert eq(da.hypot(b, a), np.hypot(y, x))
assert eq(da.sinh(a), np.sinh(x))
assert eq(da.cosh(b), np.cosh(y))
assert eq(da.tanh(a), np.tanh(x))
assert eq(da.arcsinh(b * 10), np.arcsinh(y * 10))
assert eq(da.arccosh(b * 10), np.arccosh(y * 10))
assert eq(da.arctanh(b / 10), np.arctanh(y / 10))
assert eq(da.deg2rad(a), np.deg2rad(x))
assert eq(da.rad2deg(a), np.rad2deg(x))
assert eq(da.logical_and(a < 1, b < 4), np.logical_and(x < 1, y < 4))
assert eq(da.logical_or(a < 1, b < 4), np.logical_or(x < 1, y < 4))
assert eq(da.logical_xor(a < 1, b < 4), np.logical_xor(x < 1, y < 4))
assert eq(da.logical_not(a < 1), np.logical_not(x < 1))
assert eq(da.maximum(a, 5 - a), np.maximum(a, 5 - a))
assert eq(da.minimum(a, 5 - a), np.minimum(a, 5 - a))
assert eq(da.fmax(a, 5 - a), np.fmax(a, 5 - a))
assert eq(da.fmin(a, 5 - a), np.fmin(a, 5 - a))
assert eq(da.isreal(a + 1j * b), np.isreal(x + 1j * y))
assert eq(da.iscomplex(a + 1j * b), np.iscomplex(x + 1j * y))
assert eq(da.isfinite(a), np.isfinite(x))
assert eq(da.isinf(a), np.isinf(x))
assert eq(da.isnan(a), np.isnan(x))
assert eq(da.signbit(a - 3), np.signbit(x - 3))
assert eq(da.copysign(a - 3, b), np.copysign(x - 3, y))
assert eq(da.nextafter(a - 3, b), np.nextafter(x - 3, y))
assert eq(da.ldexp(c, c), np.ldexp(z, z))
assert eq(da.fmod(a * 12, b), np.fmod(x * 12, y))
assert eq(da.floor(a * 0.5), np.floor(x * 0.5))
assert eq(da.ceil(a), np.ceil(x))
assert eq(da.trunc(a / 2), np.trunc(x / 2))
assert eq(da.degrees(b), np.degrees(y))
assert eq(da.radians(a), np.radians(x))
assert eq(da.rint(a + 0.3), np.rint(x + 0.3))
assert eq(da.fix(a - 2.5), np.fix(x - 2.5))
assert eq(da.angle(a + 1j), np.angle(x + 1j))
assert eq(da.real(a + 1j), np.real(x + 1j))
assert eq((a + 1j).real, np.real(x + 1j))
assert eq(da.imag(a + 1j), np.imag(x + 1j))
assert eq((a + 1j).imag, np.imag(x + 1j))
assert eq(da.conj(a + 1j * b), np.conj(x + 1j * y))
assert eq((a + 1j * b).conj(), (x + 1j * y).conj())
assert eq(da.clip(b, 1, 4), np.clip(y, 1, 4))
assert eq(da.fabs(b), np.fabs(y))
assert eq(da.sign(b - 2), np.sign(y - 2))
l1, l2 = da.frexp(a)
r1, r2 = np.frexp(x)
assert eq(l1, r1)
assert eq(l2, r2)
l1, l2 = da.modf(a)
r1, r2 = np.modf(x)
assert eq(l1, r1)
assert eq(l2, r2)
assert eq(da.around(a, -1), np.around(x, -1))
assert mapper is not None
self.fullname, self.unit, self.mapper, self.column, self.extras = fullname, unit, mapper, column, extras
self.conjugate = conjugate
self.axis = axis
_identity = lambda x: x
# this dict maps short axis names into full DataMapper objects
data_mappers = OrderedDict(
_=DataMapper("", "", _identity),
amp=DataMapper("Amplitude", "", abs),
logamp=DataMapper("Log-amplitude", "", lambda x: da.log10(abs(x))),
phase=DataMapper(
"Phase", "deg",
lambda x: da.arctan2(da.imag(x), da.real(x)) * 180 / math.pi),
real=DataMapper("Real", "", da.real),
imag=DataMapper("Imag", "", da.imag),
TIME=DataMapper("Time", "s", axis=0, column="TIME", mapper=_identity),
ROW=DataMapper("Row number",
"",
column=False,
axis=0,
extras=["rows"],
mapper=lambda x, rows: rows),
BASELINE=DataMapper("Baseline",
"",
column=False,
axis=0,
extras=["baselines"],
mapper=lambda x, baselines: baselines),
def image_tikhonov(self,
vis_arr,
sphere,
alpha,
scale=True,
usedask=False):
n_s = sphere.pixels.shape[0]
n_v = self.u_arr.shape[0]
lambduh = alpha / np.sqrt(n_s)
if not usedask:
gamma = self.make_gamma(sphere)
logger.info("augmented: {}".format(gamma.shape))
vis_aux = vis_to_real(vis_arr)
logger.info("vis mean: {} shape: {}".format(
np.mean(vis_aux), vis_aux.shape))
tol = min(alpha / 1e4, 1e-10)
logger.info("Solving tol={} ...".format(tol))
# reg = linear_model.ElasticNet(alpha=alpha/np.sqrt(n_s),
# tol=1e-6,
# l1_ratio = 0.01,
# max_iter=100000,
# positive=True)
if False:
(
sky,
lstop,
itn,
r1norm,
r2norm,
anorm,
acond,
arnorm,
xnorm,
var,
) = scipy.sparse.linalg.lsqr(gamma,
vis_aux,
damp=alpha,
show=True)
logger.info(
"Alpha: {}: Iterations: {}: rnorm: {}: xnorm: {}".format(
alpha, itn, r2norm, xnorm))
else:
reg = linear_model.Ridge(alpha=alpha,
tol=tol,
solver="lsqr",
max_iter=100000)
reg.fit(gamma, vis_aux)
logger.info(" Solve Complete, iter={}".format(reg.n_iter_))
sky = da.from_array(reg.coef_)
residual = vis_aux - gamma @ sky
sky, residual_norm, solution_norm = da.compute(
sky,
np.linalg.norm(residual)**2,
np.linalg.norm(sky)**2)
score = reg.score(gamma, vis_aux)
logger.info("Alpha: {}: Loss: {}: rnorm: {}: snorm: {}".format(
alpha, score, residual_norm, solution_norm))
else:
from dask_ml.linear_model import LinearRegression
import dask_glm
from dask.distributed import Client, LocalCluster
from dask.diagnostics import ProgressBar
import dask
logger.info("Starting Dask Client")
if True:
cluster = LocalCluster(dashboard_address=":8231",
processes=False)
client = Client(cluster)
else:
client = Client("tcp://localhost:8786")
logger.info("Client = {}".format(client))
harmonic_list = []
p2j = 2 * np.pi * 1.0j
dl = sphere.l
dm = sphere.m
dn = sphere.n
n_arr_minus_1 = dn - 1
du = self.u_arr
dv = self.v_arr
dw = self.w_arr
for u, v, w in zip(du, dv, dw):
harmonic = da.from_array(
np.exp(p2j * (u * dl + v * dm + w * n_arr_minus_1)) /
np.sqrt(sphere.npix),
chunks=(n_s, ),
)
harminc = client.persist(harmonic)
harmonic_list.append(harmonic)
gamma = da.stack(harmonic_list)
logger.info("Gamma Shape: {}".format(gamma.shape))
# gamma = gamma.reshape((n_v, n_s))
gamma = gamma.conj()
gamma = client.persist(gamma)
logger.info("Gamma Shape: {}".format(gamma.shape))
logger.info("Building Augmented Operator...")
proj_operator_real = da.real(gamma)
proj_operator_imag = da.imag(gamma)
proj_operator = da.block([[proj_operator_real],
[proj_operator_imag]])
proj_operator = client.persist(proj_operator)
logger.info("Proj Operator shape {}".format(proj_operator.shape))
vis_aux = da.from_array(
np.array(
np.concatenate((np.real(vis_arr), np.imag(vis_arr))),
dtype=np.float32,
))
# logger.info("Solving...")
en = dask_glm.regularizers.ElasticNet(weight=0.01)
en = dask_glm.regularizers.L2()
# dT = da.from_array(proj_operator, chunks=(-1, 'auto'))
##dT = da.from_array(proj_operator, chunks=(-1, 'auto'))
# dv = da.from_array(vis_aux)
dask.config.set({"array.chunk-size": "1024MiB"})
A = da.rechunk(proj_operator, chunks=("auto", n_s))
A = client.persist(A)
y = vis_aux # da.rechunk(vis_aux, chunks=('auto', n_s))
y = client.persist(y)
# sky = dask_glm.algorithms.proximal_grad(A, y, regularizer=en, lambduh=alpha, max_iter=10000)
logger.info("Rechunking completed.. A= {}.".format(A.shape))
reg = LinearRegression(
penalty=en,
C=1.0 / lambduh,
fit_intercept=False,
solver="lbfgs",
max_iter=1000,
tol=1e-8,
)
sky = reg.fit(A, y)
sky = reg.coef_
score = reg.score(proj_operator, vis_aux)
logger.info("Loss function: {}".format(score.compute()))
logger.info("Solving Complete: sky = {}".format(sky.shape))
sphere.set_visible_pixels(sky, scale=False)
return sky.reshape(-1, 1)
def image_tikhonov(self, vis_arr, sphere, alpha, scale=True, usedask=False):
n_s = sphere.pixels.shape[0]
n_v = self.u_arr.shape[0]
lambduh = alpha/np.sqrt(n_s)
if not usedask:
gamma = self.make_gamma(sphere)
logger.info("Building Augmented Operator...")
proj_operator_real = np.real(gamma).astype(np.float32)
proj_operator_imag = np.imag(gamma).astype(np.float32)
gamma = None
proj_operator = np.block([[proj_operator_real], [proj_operator_imag]])
proj_operator_real = None
proj_operator_imag = None
logger.info('augmented: {}'.format(proj_operator.shape))
vis_aux = np.array(np.concatenate((np.real(vis_arr), np.imag(vis_arr))), dtype=np.float32)
logger.info('vis mean: {} shape: {}'.format(np.mean(vis_aux), vis_aux.shape))
logger.info("Solving...")
reg = linear_model.ElasticNet(alpha=lambduh, l1_ratio=0.05, max_iter=10000, positive=True)
reg.fit(proj_operator, vis_aux)
sky = reg.coef_
score = reg.score(proj_operator, vis_aux)
logger.info('Loss function: {}'.format(score))
else:
from dask_ml.linear_model import LinearRegression
import dask_glm
import dask.array as da
from dask.distributed import Client, LocalCluster
from dask.diagnostics import ProgressBar
import dask
logger.info('Starting Dask Client')
if True:
cluster = LocalCluster(dashboard_address=':8231', processes=False)
client = Client(cluster)
else:
client = Client('tcp://localhost:8786')
logger.info("Client = {}".format(client))
harmonic_list = []
p2j = 2*np.pi*1.0j
dl = sphere.l
dm = sphere.m
dn = sphere.n
n_arr_minus_1 = dn - 1
du = self.u_arr
dv = self.v_arr
dw = self.w_arr
for u, v, w in zip(du, dv, dw):
harmonic = da.from_array(np.exp(p2j*(u*dl + v*dm + w*n_arr_minus_1)) / np.sqrt(sphere.npix), chunks=(n_s,))
harminc = client.persist(harmonic)
harmonic_list.append(harmonic)
gamma = da.stack(harmonic_list)
logger.info('Gamma Shape: {}'.format(gamma.shape))
#gamma = gamma.reshape((n_v, n_s))
gamma = gamma.conj()
gamma = client.persist(gamma)
logger.info('Gamma Shape: {}'.format(gamma.shape))
logger.info("Building Augmented Operator...")
proj_operator_real = da.real(gamma)
proj_operator_imag = da.imag(gamma)
proj_operator = da.block([[proj_operator_real], [proj_operator_imag]])
proj_operator = client.persist(proj_operator)
logger.info("Proj Operator shape {}".format(proj_operator.shape))
vis_aux = da.from_array(np.array(np.concatenate((np.real(vis_arr), np.imag(vis_arr))), dtype=np.float32))
#logger.info("Solving...")
en = dask_glm.regularizers.ElasticNet(weight=0.01)
en = dask_glm.regularizers.L2()
#dT = da.from_array(proj_operator, chunks=(-1, 'auto'))
##dT = da.from_array(proj_operator, chunks=(-1, 'auto'))
#dv = da.from_array(vis_aux)
dask.config.set({'array.chunk-size': '1024MiB'})
A = da.rechunk(proj_operator, chunks=('auto', n_s))
A = client.persist(A)
y = vis_aux # da.rechunk(vis_aux, chunks=('auto', n_s))
y = client.persist(y)
#sky = dask_glm.algorithms.proximal_grad(A, y, regularizer=en, lambduh=alpha, max_iter=10000)
logger.info("Rechunking completed.. A= {}.".format(A.shape))
reg = LinearRegression(penalty=en, C=1.0/lambduh,
fit_intercept=False,
solver='lbfgs',
max_iter=1000, tol=1e-8 )
sky = reg.fit(A, y)
sky = reg.coef_
score = reg.score(proj_operator, vis_aux)
logger.info('Loss function: {}'.format(score.compute()))
logger.info("Solving Complete: sky = {}".format(sky.shape))
sphere.set_visible_pixels(sky, scale=True)
return sky.reshape(-1,1)
