def setUp(self):
self.sparse = True
eqs = ['x+y','x-y']
x,y = 1,2
d,w = {}, {}
for eq in eqs: d[eq],w[eq] = eval(eq), 1.
self.ls = linsolve.LinearSolver(d,w,sparse=self.sparse)
def test_degen_sol(self):
# test how various solvers deal with degenerate solutions
d = {'x+y': 1., '2*x+2*y': 2.}
ls = linsolve.LinearSolver(d, sparse=self.sparse)
for mode in ('pinv', 'lsqr'):
sol = ls.solve(mode=mode)
np.testing.assert_almost_equal(sol['x'] + sol['y'], 1., 6)
with pytest.raises(np.linalg.LinAlgError):
ls.solve(mode='solve')
def _build_solver(self, noise=None, **kwargs):
"""
Builds linsolve solver
"""
if not noise is None:
N = self.get_noise_matrix(noise_type=kwargs['noise_type'])
self.ls = linsolve.LinearSolverNoise(self.eqs, N, **self.consts)
else:
self.ls = linsolve.LinearSolver(self.eqs, **self.consts)
def test_nonunity_wgts(self):
x,y = 1.,2.
a,b = 3.*np.ones(4),1.
eqs = ['a*x+y','x+b*y']
d,w = {}, {}
for eq in eqs: d[eq],w[eq] = eval(eq)*np.ones(4), 2*np.ones(4)
ls = linsolve.LinearSolver(d,w,a=a,b=b, sparse=self.sparse)
sol = ls.solve()
np.testing.assert_almost_equal(sol['x'], x*np.ones(4,dtype=np.float))
np.testing.assert_almost_equal(sol['y'], y*np.ones(4,dtype=np.float))
def test_solve_arrays(self):
x = np.arange(100,dtype=np.float); x.shape = (10,10)
y = np.arange(100,dtype=np.float); y.shape = (10,10)
eqs = ['2*x+y','-x+3*y']
d,w = {}, {}
for eq in eqs: d[eq],w[eq] = eval(eq), 1.
ls = linsolve.LinearSolver(d,w, sparse=self.sparse)
sol = ls.solve()
np.testing.assert_almost_equal(sol['x'], x)
np.testing.assert_almost_equal(sol['y'], y)
def test_const_arrays(self):
x,y = 1.,2.
a = np.array([3.,4,5])
b = np.array([1.,2,3])
eqs = ['a*x+y','x+b*y']
d,w = {}, {}
for eq in eqs: d[eq],w[eq] = eval(eq), 1.
ls = linsolve.LinearSolver(d,w,a=a,b=b, sparse=self.sparse)
sol = ls.solve()
np.testing.assert_almost_equal(sol['x'], x*np.ones(3,dtype=np.float))
np.testing.assert_almost_equal(sol['y'], y*np.ones(3,dtype=np.float))
def test_chisq(self):
x = 1.
d = {'x':1, 'a*x':2}
ls = linsolve.LinearSolver(d,a=1.0, sparse=self.sparse)
sol = ls.solve()
chisq = ls.chisq(sol)
np.testing.assert_equal(chisq, .5)
x = 1.
d = {'x':1, '1.0*x':2}
ls = linsolve.LinearSolver(d, sparse=self.sparse)
sol = ls.solve()
chisq = ls.chisq(sol)
np.testing.assert_equal(chisq, .5)
x = 1.
d = {'1*x': 2.0, 'x': 1.0}
w = {'1*x': 1.0, 'x': .5}
ls = linsolve.LinearSolver(d, wgts=w, sparse=self.sparse)
sol = ls.solve()
chisq = ls.chisq(sol)
self.assertAlmostEqual(sol['x'], 5.0/3.0)
self.assertAlmostEqual(ls.chisq(sol), 1.0/3.0)
def test_dtypes(self):
ls = linsolve.LinearSolver({'x_': 1.0+1.0j}, sparse=self.sparse)
self.assertEqual(ls.dtype,np.float32)
self.assertEqual(type(ls.solve()['x']), np.complex64)
ls = linsolve.LinearSolver({'x': 1.0+1.0j}, sparse=self.sparse)
self.assertEqual(ls.dtype, np.complex64)
self.assertEqual(type(ls.solve()['x']), np.complex64)
ls = linsolve.LinearSolver({'x_': np.ones(1,dtype=np.complex64)[0]}, sparse=self.sparse)
self.assertEqual(ls.dtype,np.float32)
self.assertEqual(type(ls.solve()['x']), np.complex64)
ls = linsolve.LinearSolver({'x': np.ones(1,dtype=np.complex64)[0]}, sparse=self.sparse)
self.assertEqual(ls.dtype,np.complex64)
self.assertEqual(type(ls.solve()['x']), np.complex64)
ls = linsolve.LinearSolver({'c*x': 1.0}, c=1.0+1.0j, sparse=self.sparse)
self.assertEqual(ls.dtype,np.complex64)
self.assertEqual(type(ls.solve()['x']), np.complex64)
d = {'c*x': np.ones(1,dtype=np.float32)[0]}
wgts = {'c*x': np.ones(1,dtype=np.float64)[0]}
c = np.ones(1,dtype=np.float32)[0]
ls = linsolve.LinearSolver(d, wgts=wgts, c=c, sparse=self.sparse)
self.assertEqual(ls.dtype,np.float64)
self.assertEqual(type(ls.solve()['x']), np.float64)
def test_solve_arrays(self):
# range of 1 to 101 prevents "The exact solution is x = 0" printouts
x = np.arange(1, 101, dtype=np.float64)
x.shape = (10, 10)
y = np.arange(1, 101, dtype=np.float64)
y.shape = (10, 10)
eqs = ['2*x+y', '-x+3*y']
d, w = {}, {}
for eq in eqs:
d[eq], w[eq] = eval(eq), 1.
ls = linsolve.LinearSolver(d, w, sparse=self.sparse)
sol = ls.solve()
np.testing.assert_almost_equal(sol['x'], x)
np.testing.assert_almost_equal(sol['y'], y)
def test_eval(self):
x,y = 1.,2.
a,b = 3.*np.ones(4),1.
eqs = ['a*x+y','x+b*y']
d,w = {}, {}
for eq in eqs: d[eq],w[eq] = eval(eq)*np.ones(4), np.ones(4)
ls = linsolve.LinearSolver(d,w,a=a,b=b, sparse=self.sparse)
sol = ls.solve()
np.testing.assert_almost_equal(sol['x'], x*np.ones(4,dtype=np.float))
np.testing.assert_almost_equal(sol['y'], y*np.ones(4,dtype=np.float))
result = ls.eval(sol)
for eq in d:
np.testing.assert_almost_equal(d[eq], result[eq])
result = ls.eval(sol, 'a*x+b*y')
np.testing.assert_almost_equal(3*1+1*2, list(result.values())[0])
def test_dtypes(self):
ls = linsolve.LinearSolver({'x_': 1.0 + 1.0j}, sparse=self.sparse)
# conjugation should trigger re_im_split, splitting the
# complex64 type into two float32 types
assert ls.dtype == np.float32
assert type(ls.solve()['x']) == np.complex64
ls = linsolve.LinearSolver({'x': 1.0 + 1.0j}, sparse=self.sparse)
assert ls.dtype == np.complex64
assert type(ls.solve()['x']) == np.complex64
ls = linsolve.LinearSolver({'x_': np.ones(1, dtype=np.complex64)[0]},
sparse=self.sparse)
# conjugation should trigger re_im_split, splitting the
# complex64 type into two float32 types
assert ls.dtype == np.float32
assert type(ls.solve()['x']) == np.complex64
ls = linsolve.LinearSolver({'x': np.ones(1, dtype=np.complex64)[0]},
sparse=self.sparse)
assert ls.dtype, np.complex64
assert type(ls.solve()['x']) == np.complex64
ls = linsolve.LinearSolver({'c*x': np.array(1.0, dtype=np.float32)},
c=1.0 + 1.0j,
sparse=self.sparse)
assert ls.dtype == np.complex64
assert type(ls.solve()['x']) == np.complex64
d = {'c*x': np.ones(1, dtype=np.float32)[0]}
wgts = {'c*x': np.ones(1, dtype=np.float64)[0]}
c = np.ones(1, dtype=np.float32)[0]
ls = linsolve.LinearSolver(d, wgts=wgts, c=c, sparse=self.sparse)
assert ls.dtype == np.float64
assert type(ls.solve()['x']) == np.float64
d = {'c*x': np.ones(1, dtype=np.float32)[0]}
wgts = {'c*x': np.ones(1, dtype=np.float32)[0]}
c = np.ones(1, dtype=np.float32)[0]
ls = linsolve.LinearSolver(d, wgts=wgts, c=c, sparse=self.sparse)
assert ls.dtype == np.float32
assert type(ls.solve()['x']) == np.float32
def test_A_shape(self):
# range of 1 to 11 prevents "The exact solution is x = 0" printouts
consts = {'a': np.arange(1, 11), 'b': np.zeros((1, 10))}
ls = linsolve.LinearSolver({'a*x+b*y': 0.}, {'a*x+b*y': 1}, **consts)
assert ls._A_shape() == (1, 2, 10 * 10)
import numpy as np
import time, random
np.random.seed(0)
NPRMS = 10
NEQS = 100
SIZE = 100000
MODE = 'solve' # sparse: 0.43 s
#MODE = 'lsqr' # sparse: 3.8 s
#MODE = 'pinv' # sparse: 2.72 s
prms = {'g%d' % i: np.arange(SIZE) for i in range(NPRMS)}
prm_list = list(prms.keys())
prms['c0'] = np.arange(SIZE)
eqs = [('+c0*'.join(['%s'] * 2)) % tuple(random.sample(prm_list, 2))
for i in range(NEQS)]
data = {eq: eval(eq, prms) for eq in eqs}
ls = linsolve.LinearSolver(data, c0=prms['c0'], sparse=True)
t0 = time.time()
sol = ls.solve(mode=MODE)
t1 = time.time()
print('Solved in {}'.format(t1 - t0))
for k in prm_list:
assert np.mean(np.abs(sol[k] - prms[k])**2) < 1e-3
def fit_data(self, all_chans=True, ch=600, calc_fit_err=False,
absolute_sigma = False):
"""
Fit gains and receiver temperatures based on LST evolution of signal fit to
simulated Tsky.
Args:
all_chans: (bool) fit all channels if set to True (default, slow).
Otherwise only fit channel ch (faster).
ch: (int) Only fit this channel number. Default 600.
Ignored if all_chans == True.
calc_fit_err: (bool) Calculate the covariance matrix of the fitted
parameter. Default False.
"""
self.gains = {}
self.Trxr = {}
self.fits = {}
self.fit_cov = {}
for poli, pol in enumerate(self.pols):
for ant in self.ants:
d_ls = {}
w_ls = {}
kwargs = {}
if self.use_npz:
data = self.data[poli, ant, :, :]
flags = np.zeros_like(data)
freq_low = np.where(self.freqs == np.min(self.freqs))[0][0]
freq_high = np.where(self.freqs == np.max(self.freqs))[0][0]
else:
data = np.abs(self.uv.get_data((ant, ant, self.rev_pol_map[pol])))
flags = self.uv.get_flags((ant, ant, self.rev_pol_map[pol]))
freq_low = np.where(self.uv.freq_array*1e-6 == np.min(self.freqs))[1][0]
freq_high = np.where(self.uv.freq_array*1e-6 == np.max(self.freqs))[1][0]
for i in range(self.lsts.size):
if all_chans:
# Solve for all channels at once
d_ls['Tsky%d*g+n' % i] = data[i, freq_low:(freq_high+1)]
w_ls['Tsky%d*g+n' % i] = 1 - flags[i, freq_low:(freq_high+1)]
kwargs['Tsky%d' % i] = self.Tsky[poli, i, :] - self.Tsky_mean[poli]
else:
# Only solve channel ch
d_ls['Tsky%d*g+n' % i] = data[i, ch]
w_ls['Tsky%d*g+n' % i] = 1 - flags[i, ch]
kwargs['Tsky%d' % i] = self.Tsky[poli, i, ch] - self.Tsky_mean[poli]
ls = linsolve.LinearSolver(d_ls, w_ls, **kwargs)
sol = ls.solve()
self.fits[(ant, pol)] = (sol['g'], sol['n'])
self._fits2gTrxr(all_chans=all_chans, ch=ch)
if calc_fit_err and all_chans:
for poli, pol in enumerate(self.pols):
for ant in self.ants:
cov_mat = np.zeros(((freq_high+1)-freq_low, 2, 2))
for fi, freq in enumerate(np.arange(freq_low,(freq_high+1))):
Tsky_prime = self.Tsky[poli, :, freq] - self.Tsky_mean[poli, freq]
A = np.column_stack([Tsky_prime, np.ones_like(Tsky_prime)])
Q_inv = np.linalg.inv(np.matmul(A.T, A))
if absolute_sigma:
cov_mat[fi,:,:] = Q_inv
else:
yhat = self.fits[(ant, pol)][0][freq]*Tsky_prime+self.fits[(ant, pol)][1][freq]
rss = np.sum((data[:, freq] - yhat) ** 2)
cov_mat[fi,:,:] = rss * Q_inv / (Tsky_prime.shape[0] - 2)
self.fit_cov[(ant,pol)] = cov_mat
self._calc_Trxr_err()
elif calc_fit_err:
for poli, pol in enumerate(self.pols):
for ant in self.ants:
Tsky_prime = self.Tsky[poli, :, ch] - self.Tsky_mean[poli, ch]
A = np.column_stack([Tsky_prime, np.ones_like(Tsky_prime)])
Q_inv = np.linalg.inv(np.matmul(A.T, A))
if absolute_sigma:
self.fit_cov[(ant,pol)] = Q_inv
else:
yhat = self.fits[(ant, pol)][0]*Tsky_prime+self.fits[(ant, pol)][1]
rss = np.sum((data[:, ch] - yhat) ** 2)
self.fit_cov[(ant,pol)] = rss * Q_inv / (Tsky_prime.shape[0] - 2)
self._calc_Trxr_err()
inverted A matrix to be reused. In this case, we benchmark the
case when a sparse representation of A is used.'''
import linsolve
import numpy as np
import time, random
np.random.seed(0)
NPRMS = 10
NEQS = 100
SIZE = 1000000
# sparse: 0.82 s
prms = {'g%d' % i: np.arange(SIZE) for i in range(NPRMS)}
prm_list = list(prms.keys())
eqs = [('+'.join(['%s'] * 3)) % tuple(random.sample(prm_list, 3))
for i in range(NEQS)]
data = {eq: eval(eq, prms) for eq in eqs}
ls = linsolve.LinearSolver(data, sparse=True)
t0 = time.time()
sol = ls.solve()
t1 = time.time()
print('Solved in {}'.format(t1 - t0))
for k in prm_list:
assert np.mean(np.abs(sol[k] - prms[k])**2) < 1e-3
parameters and a large number of parallel instances that allow the
inverted A matrix to be reused.'''
import linsolve
import numpy as np
import time, random
np.random.seed(0)
NPRMS = 10
NEQS = 100
SIZE = 1000000
# dense: 0.48 s
prms = {'g%d' % i: np.arange(SIZE) for i in range(NPRMS)}
prm_list = list(prms.keys())
eqs = [('+'.join(['%s'] * 5)) % tuple(random.sample(prm_list, 5))
for i in range(NEQS)]
data = {eq: eval(eq, prms) for eq in eqs}
ls = linsolve.LinearSolver(data, sparse=False)
t0 = time.time()
sol = ls.solve()
t1 = time.time()
print('Solved in {}'.format(t1-t0))
for k in prm_list:
assert np.mean(np.abs(sol[k] - prms[k])**2) < 1e-3
def test_A_shape(self):
consts = {'a':np.arange(10), 'b':np.zeros((1,10))}
ls = linsolve.LinearSolver({'a*x+b*y':0.},{'a*x+b*y':1},**consts)
self.assertEqual(ls._A_shape(), (1,2,10*10))
