def __init__(self, x_wc, K):
self.x_wc = x_wc
self.x_cw = coord_xfms.ssc.inverse(self.x_wc)
# camera calibration matrix
self.K = K
# camera center
self.C = self.x_wc[0:3]
# transforms points in world frame to points in camera frame
self.wHc = coord_xfms.xyzrph2matrix(self.x_cw)
# transforms points in camera frame to points in world frame
self.cHw = coord_xfms.xyzrph2matrix(self.x_wc)
# the stupid [R t] notation that I absolutely hate
self.Rt = self.wHc[0:3, :]
# projects points in world frame to image plane of camera
self.P = K.dot(self.Rt)
# used for normalized image points
self.invK = pl.inv(K[0:3, 0:3])
# used in back-projecting points to rays
self.pinvP = pl.pinv(self.P)
def __init__ (self, x_wc, K):
self.x_wc = x_wc
self.x_cw = coord_xfms.ssc.inverse (self.x_wc)
# camera calibration matrix
self.K = K
# camera center
self.C = self.x_wc[0:3]
# transforms points in world frame to points in camera frame
self.wHc = coord_xfms.xyzrph2matrix (self.x_cw)
# transforms points in camera frame to points in world frame
self.cHw = coord_xfms.xyzrph2matrix (self.x_wc)
# the stupid [R t] notation that I absolutely hate
self.Rt = self.wHc[0:3,:]
# projects points in world frame to image plane of camera
self.P = K.dot (self.Rt)
# used for normalized image points
self.invK = pl.inv (K[0:3,0:3])
# used in back-projecting points to rays
self.pinvP = pl.pinv (self.P)
def find_factors(idat, odat, k = None):
"""
A routine to compute the main predictors (linear combinations of
coordinates) in idat to predict odat.
*Parameters*
idat: d x n data matrix,
with n measurements, each of dimension d
odat: q x n data matrix
with n measurements, each of dimension q
*Returns*
**Depending on whether or not** *k* **is provided, the returned
value is different**
* if k is given, compute the first k regressors and return an orthogonal
matrix that contains the regressors in its colums,
i.e. reg[0,:] is the first regressor
* if k is not given or None, return a d-dimensional vector v(k)
explaining which fraction of the total predictable variance can be
explained using only k regressors.
**NOTE**
#. idat and odat must have zero mean
#. To interpret the regressors, it is advisable to have the
for columns of idat having the same variance
"""
# transform into z-scores
u, s, v = svd(idat, full_matrices = False)
su = dot(diag(1./s), u.T)
z = dot(su,idat)
# ! Note that the covariance of z is *NOT* 1, but 1/n; z*z.T = 1 !
# least-squares regression:
A = dot(odat, pinv(z))
uA, sigma_A, vA = svd(A, full_matrices = False)
if k is None:
vk = cumsum(sigma_A**2) / sum(sigma_A**2)
return vk
else:
# choose k predictors
sigma_A1 = sigma_A.copy()
sigma_A1[k:] = 0
A1 = reduce(dot, [uA, diag(sigma_A1), vA])
B = dot(A1, su)
uB, sigma_B, vB = svd(B, full_matrices = False)
regs = vB[:k,:].T
return regs
def _calculate_svd(pp, r_0, beta, N_piercepoints):
"""
Returns result (U) of svd for K-L vectors
Parameters
----------
pp : array
Array of piercepoint locations
r_0: float
Scale size of amp fluctuations (m)
beta: float
Power-law index for amp structure function (5/3 => pure Kolmogorov
turbulence)
N_piercepoints : int
Number of piercepoints
Returns
-------
C : array
C matrix
pinvC : array
Inv(C) matrix
U : array
Unitary matrix
"""
import numpy as np
from pylab import kron, concatenate, pinv, norm, newaxis, find, amin, svd, eye
D = np.resize(pp, (N_piercepoints, N_piercepoints, 3))
D = np.transpose(D, (1, 0, 2)) - D
D2 = np.sum(D**2, axis=2)
C = -(D2 / r_0**2)**(beta / 2.0) / 2.0
pinvC = pinv(C, rcond=1e-3)
U, S, V = svd(C)
return C, pinvC, U
def _calculate_svd(pp, r_0, beta, N_piercepoints):
"""
Returns result (U) of svd for K-L vectors
Parameters
----------
pp : array
Array of piercepoint locations
r_0: float
Scale size of amp fluctuations (m)
beta: float
Power-law index for amp structure function (5/3 => pure Kolmogorov
turbulence)
N_piercepoints : int
Number of piercepoints
Returns
-------
C : array
C matrix
pinvC : array
Inv(C) matrix
U : array
Unitary matrix
"""
import numpy as np
from pylab import kron, concatenate, pinv, norm, newaxis, find, amin, svd, eye
D = np.resize(pp, (N_piercepoints, N_piercepoints, 3))
D = np.transpose(D, (1, 0, 2)) - D
D2 = np.sum(D**2, axis=2)
C = -(D2 / r_0**2)**(beta / 2.0) / 2.0
pinvC = pinv(C, rcond=1e-3)
U, S, V = svd(C)
return C, pinvC, U
def makeTECparmdb(H, solset, TECsolTab, timewidths, freq, freqwidth):
"""Returns TEC screen parmdb parameters
H - H5parm object
solset - solution set with TEC screen parameters
TECsolTab = solution table with tecscreen values
timewidths - time widths of output parmdb
freq - frequency of output parmdb
freqwidth - frequency width of output parmdb
"""
from pylab import pinv
global ipbar, pbar
station_dict = H.getAnt(solset)
station_names = station_dict.keys()
station_positions = station_dict.values()
source_dict = H.getSou(solset)
source_names = source_dict.keys()
source_positions = source_dict.values()
tec_sf = solFetcher(TECsolTab)
tec_screen, axis_vals = tec_sf.getValues()
times = axis_vals['time']
beta = TECsolTab._v_attrs['beta']
r_0 = TECsolTab._v_attrs['r_0']
height = TECsolTab._v_attrs['height']
order = TECsolTab._v_attrs['order']
pp = tec_sf.t.piercepoint
N_sources = len(source_names)
N_times = len(times)
N_freqs = 1
N_stations = len(station_names)
N_piercepoints = N_sources * N_stations
freqs = freq
freqwidths = freqwidth
parms = {}
v = {}
v['times'] = times
v['timewidths'] = timewidths
v['freqs'] = freqs
v['freqwidths'] = freqwidths
for station_name in station_names:
for source_name in source_names:
v['values'] = np.zeros((N_times, N_freqs), dtype=np.double)
parmname = 'Piercepoint:X:%s:%s' % (station_name, source_name)
parms[parmname] = v.copy()
v['values'] = np.zeros((N_times, N_freqs), dtype=np.double)
parmname = 'Piercepoint:Y:%s:%s' % (station_name, source_name)
parms[parmname] = v.copy()
v['values'] = np.zeros((N_times, N_freqs), dtype=np.double)
parmname = 'Piercepoint:Z:%s:%s' % (station_name, source_name)
parms[parmname] = v.copy()
v['values'] = np.zeros((N_times, N_freqs), dtype=np.double)
parmname = 'TECfit_white:%s:%s' % (station_name, source_name)
parms[parmname] = v.copy()
v['values'] = np.zeros((N_times, N_freqs), dtype=np.double)
parmname = 'TECfit_white:0:%s:%s' % (station_name, source_name)
parms[parmname] = v.copy()
v['values'] = np.zeros((N_times, N_freqs), dtype=np.double)
parmname = 'TECfit_white:1:%s:%s' % (station_name, source_name)
parms[parmname] = v.copy()
for k in range(N_times):
D = np.resize(pp[k, :, :], (N_piercepoints, N_piercepoints, 3))
D = np.transpose(D, (1, 0, 2)) - D
D2 = np.sum(D**2, axis=2)
C = -(D2 / (r_0**2))**(beta / 2.0) / 2.0
tec_fit_white = np.dot(pinv(C),
tec_screen[:, k, :].reshape(N_piercepoints))
pp_idx = 0
for src, source_name in enumerate(source_names):
for sta, station_name in enumerate(station_names):
parmname = 'Piercepoint:X:%s:%s' % (station_name, source_name)
parms[parmname]['values'][k, 0] = pp[k, pp_idx, 0]
parmname = 'Piercepoint:Y:%s:%s' % (station_name, source_name)
parms[parmname]['values'][k, 0] = pp[k, pp_idx, 1]
parmname = 'Piercepoint:Z:%s:%s' % (station_name, source_name)
parms[parmname]['values'][k, 0] = pp[k, pp_idx, 2]
parmname = 'TECfit_white:%s:%s' % (station_name, source_name)
parms[parmname]['values'][k, 0] = tec_fit_white[pp_idx]
parmname = 'TECfit_white:0:%s:%s' % (station_name, source_name)
parms[parmname]['values'][k, 0] = tec_fit_white[pp_idx]
parmname = 'TECfit_white:1:%s:%s' % (station_name, source_name)
parms[parmname]['values'][k, 0] = tec_fit_white[pp_idx]
pp_idx += 1
pbar.update(ipbar)
ipbar += 1
time_start = times[0] - timewidths[0] / 2
time_end = times[-1] + timewidths[-1] / 2
v['times'] = np.array([(time_start + time_end) / 2])
v['timewidths'] = np.array([time_end - time_start])
v_r0 = v.copy()
v_r0['values'] = np.array(r_0, dtype=np.double, ndmin=2)
parms['r_0'] = v_r0
v_beta = v.copy()
v_beta['values'] = np.array(beta, dtype=np.double, ndmin=2)
parms['beta'] = v_beta
v_height = v.copy()
v_height['values'] = np.array(height, dtype=np.double, ndmin=2)
parms['height'] = v_height
return parms
def add_stations(station_selection,
phases0,
phases1,
flags,
mask,
station_names,
station_positions,
source_names,
source_selection,
times,
freqs,
r,
nband_min=2,
soln_type='phase',
nstations_max=None,
excluded_stations=None,
t_step=5,
tec_step1=5,
tec_step2=21,
search_full_tec_range=False):
"""
Adds stations to TEC fitting using an iterative initial-guess search to
ensure the global min is found
Keyword arguments:
station_selection -- indices of stations to use in fitting
phases0 -- XX phase solutions
phases1 -- YY phase solutions
flags -- phase solution flags (0 = use, 1 = flagged)
mask -- mask for sources and frequencies (0 = ignore, 1 = use)
station_names -- array of station names
source_names -- array of source names
source_selection -- indices of sources to use in fitting
times -- array of times
freqs -- array of frequencies
r -- array of TEC solutions returned by fit_tec_per_source_pair()
nband_min -- min number of bands for a source to be used
soln_type -- type of phase solution: 'phase' or 'scalarphase'
nstations_max -- max number of stations to use
excluded_stations -- stations to exclude
t_step -- try full TEC range every t_step number of solution times
tec_step1 -- number of steps in TEC subrange (+/- last TEC fit value)
tec_step2 -- number of steps in full TEC range (-0.1 -- 0.1)
search_full_tec_range -- always search the full TEC range (-0.1 -- 0.1)
"""
from pylab import pinv, newaxis, find, amin
import numpy as np
from lofar.expion import baselinefitting
import progressbar
N_sources_selected = len(source_selection)
N_stations_selected = len(station_selection)
N_piercepoints = N_sources_selected * N_stations_selected
N_times = len(times)
N_stations = len(station_names)
N_sources = len(source_names)
N_pairs = 0
for ii, i in enumerate(source_selection):
for jj, j in enumerate(source_selection):
if j == i:
break
subband_selection = find(mask[i, :] * mask[j, :])
if len(subband_selection) < nband_min:
continue
N_pairs += 1
D = np.resize(station_positions, (N_stations, N_stations, 3))
D = np.transpose(D, (1, 0, 2)) - D
D = np.sqrt(np.sum(D**2, axis=2))
station_selection1 = station_selection
stations_to_add = np.array([
i for i in range(len(station_names)) if i not in station_selection1
and station_names[i] not in excluded_stations
])
if len(stations_to_add) == 0:
return station_selection1, r
# Check if desired number of stations is already reached
if nstations_max is not None:
if len(station_selection1) >= nstations_max:
return station_selection1, r
logging.info("Using fitting with iterative search for remaining stations "
"(up to {0} stations in total)".format(nstations_max))
q = r
while len(stations_to_add) > 0:
D1 = D[stations_to_add[:, newaxis], station_selection1[newaxis, :]]
minimum_distance = amin(D1, axis=1)
station_to_add = stations_to_add[np.argmin(minimum_distance)]
station_selection1 = np.append(station_selection1, station_to_add)
N_stations_selected1 = len(station_selection1)
# Remove station from list
stations_to_add = stations_to_add[stations_to_add != station_to_add]
sols_list = []
eq_list = []
min_e_list = []
ipbar = 0
logging.info('Fitting TEC values with {0} included...'.format(
station_names[station_to_add]))
pbar = progressbar.ProgressBar(maxval=N_pairs * N_times).start()
for ii, i in enumerate(source_selection):
for jj, j in enumerate(source_selection):
if j == i:
break
subband_selection = find(mask[i, :] * mask[j, :])
if len(subband_selection) < nband_min:
continue
logging.debug('Adding {0} for source pair: {1}-{2}'.format(
station_names[station_to_add], i, j))
p0 = phases0[i, station_selection1[:, newaxis],
subband_selection[newaxis, :], :] - phases0[
j, station_selection1[:, newaxis],
subband_selection[newaxis, :], :]
p0 = p0 - np.mean(p0, axis=0)[newaxis, :, :]
if soln_type != 'scalarphase':
p1 = phases1[i, station_selection1[:, newaxis],
subband_selection[newaxis, :], :] - phases1[
j, station_selection1[:, newaxis],
subband_selection[newaxis, :], :]
p1 = p1 - np.mean(p1, axis=0)[newaxis, :, :]
A = np.zeros((len(subband_selection), 1))
A[:, 0] = 8.44797245e9 / freqs[subband_selection]
flags_source_pair = flags[
i, station_selection1[:, newaxis],
subband_selection[newaxis, :], :] * flags[
j, station_selection1[:, newaxis],
subband_selection[newaxis, :], :]
constant_parms = np.zeros((1, N_stations_selected1),
dtype=np.bool)
sols = np.zeros((N_times, N_stations_selected1),
dtype=np.float)
p_0_best = None
for t_idx in range(N_times):
if np.mod(t_idx, t_step) == 0:
min_e = np.Inf
if p_0_best is not None and not search_full_tec_range:
min_tec = p_0_best[0, -1] - 0.02
max_tec = p_0_best[0, -1] + 0.02
nsteps = tec_step1
else:
min_tec = -0.1
max_tec = 0.1
nsteps = tec_step2
logging.debug(
' Trying initial guesses between {0} and '
'{1} TECU'.format(min_tec, max_tec))
for offset in np.linspace(min_tec, max_tec, nsteps):
p_0 = np.zeros((1, N_stations_selected1),
np.double)
p_0[0, :N_stations_selected1 -
1] = (q[ii, t_idx, :] -
q[jj, t_idx, :])[newaxis, :]
p_0[0, -1] = offset
x = p0[:, :, t_idx].copy()
f = flags_source_pair[:, :, t_idx].copy()
sol0 = baselinefitting.fit(x.T, A, p_0, f,
constant_parms)
sol0 -= np.mean(sol0)
residual = np.mod(
np.dot(A, sol0) - x.T + np.pi,
2 * np.pi) - np.pi
residual = residual[f.T == 0]
e = np.var(residual)
if soln_type != 'scalarphase':
x = p1[:, :, t_idx].copy()
f = flags_source_pair[:, :, t_idx].copy()
sol1 = baselinefitting.fit(
x.T, A, p_0, f, constant_parms)
sol1 -= np.mean(sol1)
residual = np.mod(
np.dot(A, sol1) - x.T + np.pi,
2 * np.pi) - np.pi
residual = residual[f.T == 0]
e += np.var(residual)
else:
sol1 = sol0
if e < min_e:
logging.debug(
' Found new min variance of {0} '
'with initial guess of {1} TECU'.format(
e, p_0[0, -1]))
min_e = e
p_0_best = p_0
sols[t_idx, :] = (sol0[0, :] + sol1[0, :]) / 2
else:
# Use previous init
x = p0[:, :, t_idx].copy()
f = flags_source_pair[:, :, t_idx].copy()
sol0 = baselinefitting.fit(x.T, A, p_0_best, f,
constant_parms)
sol0 -= np.mean(sol0)
if soln_type != 'scalarphase':
x = p1[:, :, t_idx].copy()
f = flags_source_pair[:, :, t_idx].copy()
sol1 = baselinefitting.fit(x.T, A, p_0_best, f,
constant_parms)
sol1 -= np.mean(sol1)
else:
sol1 = sol0
sols[t_idx, :] = (sol0[0, :] + sol1[0, :]) / 2
ipbar += 1
pbar.update(ipbar)
### Remove outliers
logging.debug(' Searching for outliers...')
for kk in range(10):
s = sols[:, -1].copy()
selection = np.zeros(len(s), np.bool)
for t_idx in range(len(s)):
start_idx = np.max([t_idx - 10, 0])
end_idx = np.min([t_idx + 10, len(s)])
selection[t_idx] = np.sum(
abs(s[start_idx:end_idx] - s[t_idx]) < 0.02) > (
end_idx - start_idx - 8)
outliers = find(np.logical_not(selection))
if len(outliers) == 0:
break
for t_idx in outliers:
try:
idx0 = find(selection[:t_idx])[-1]
except IndexError:
idx0 = -1
try:
idx1 = find(selection[t_idx + 1:])[0] + t_idx + 1
except IndexError:
idx1 = -1
if idx0 == -1:
s[t_idx] = s[idx1]
elif idx1 == -1:
s[t_idx] = s[idx0]
else:
s[t_idx] = (s[idx0] * (idx1 - t_idx) + s[idx1] *
(t_idx - idx0)) / (idx1 - idx0)
p_0 = np.zeros((1, N_stations_selected1), np.double)
p_0[0, :] = sols[t_idx, :]
p_0[0, -1] = s[t_idx]
x = p0[:, :, t_idx].copy()
f = flags_source_pair[:, :, t_idx].copy()
sol0 = baselinefitting.fit(x.T, A, p_0, f,
constant_parms)
sol0 -= np.mean(sol0)
if soln_type != 'scalarphase':
x = p1[:, :, t_idx].copy()
sol1 = baselinefitting.fit(x.T, A, p_0, f,
constant_parms)
sol1 -= np.mean(sol1)
else:
sol1 = sol0
sols[t_idx, :] = (sol0[0, :] + sol1[0, :]) / 2
weight = 1.0
sols_list.append(weight * sols)
min_e_list.append(min_e)
eq = np.zeros(N_sources)
eq[ii] = weight
eq[jj] = -weight
eq_list.append(eq)
sols = np.array(sols_list)
B = np.array(eq_list)
pinvB = pinv(B)
q = np.dot(pinvB, sols.transpose([1, 0, 2]))
pbar.finish()
if nstations_max is not None:
if N_stations_selected1 == nstations_max:
break
return station_selection1, q
def fit_tec_per_source_pair(phases,
flags,
mask,
freqs,
init_sols=None,
init_sols_per_pair=False,
propagate=False,
nband_min=2):
"""Fits TEC values to phase solutions per source pair
Returns TEC solutions as array of shape (N_sources, N_times, N_stations)
Keyword arguments:
phases -- array of phase solutions
flags -- phase solution flags (0 = use, 1 = flagged)
mask -- mask for sources and frequencies (0 = ignore, 1 = use)
freqs -- array of frequencies
init_sols -- solutions to use to initialize the fits
init_sols_per_pair -- init_sols are per source pair (not per source)
propagate -- propagate solutions from previous solution
nband_min -- min number of bands for a source to be used
"""
from pylab import pinv, newaxis, find
import numpy as np
from lofar.expion import baselinefitting
import progressbar
sols_list = []
eq_list = []
vars_list = []
var_eq_list = []
N_sources = phases.shape[0]
N_stations = phases.shape[1]
N_times = phases.shape[3]
N_pairs = 0
for i in range(N_sources):
for j in range(i):
subband_selection = find(mask[i, :] * mask[j, :])
if len(subband_selection) < nband_min:
continue
N_pairs += 1
if init_sols is None and not init_sols_per_pair:
init_sols = np.zeros((N_sources, N_times, N_stations), dtype=np.float)
elif init_sols is None and init_sols_per_pair:
init_sols = np.zeros((N_pairs, N_times, N_stations), dtype=np.float)
source_pairs = []
k = 0
ipbar = 0
logging.info('Fitting TEC values...')
pbar = progressbar.ProgressBar(maxval=N_pairs * N_times).start()
for i in range(N_sources):
for j in range(i):
subband_selection = find(mask[i, :] * mask[j, :])
if len(subband_selection) < nband_min:
continue
source_pairs.append((i, j))
p = phases[i, :,
subband_selection, :] - phases[j, :,
subband_selection, :]
A = np.zeros((len(subband_selection), 1))
A[:, 0] = 8.44797245e9 / freqs[subband_selection]
flags_source_pair = flags[i, :, subband_selection, :] * flags[
j, :, subband_selection, :]
constant_parms = np.zeros((1, N_stations), dtype=np.bool)
sols = np.zeros((N_times, N_stations), dtype=np.float)
if init_sols_per_pair:
p_0 = init_sols[k, 0, :][newaxis, :]
else:
p_0 = (init_sols[i, 0, :] - init_sols[j, 0, :])[newaxis, :]
for t_idx in range(N_times):
x = p[:, :, t_idx].copy()
f = flags_source_pair[:, :, t_idx].copy()
if not propagate:
if init_sols_per_pair:
p_0 = init_sols[k, t_idx, :][newaxis, :]
else:
p_0 = (init_sols[i, t_idx, :] -
init_sols[j, 0, :])[newaxis, :]
sol = baselinefitting.fit(x, A, p_0, f, constant_parms)
if propagate:
p_0 = sol.copy()
sols[t_idx, :] = sol[0, :]
ipbar += 1
pbar.update(ipbar)
sols = sols[:, :] - np.mean(sols[:, :], axis=1)[:, newaxis]
weight = len(subband_selection)
sols_list.append(sols * weight)
eq = np.zeros(N_sources)
eq[i] = weight
eq[j] = -weight
eq_list.append(eq)
k += 1
pbar.finish()
sols = np.array(sols_list)
B = np.array(eq_list)
source_selection = find(np.sum(abs(B), axis=0))
N_sources = len(source_selection)
if N_sources == 0:
logging.error(
'All sources have fewer than the required minimum number of bands.'
)
return None, None
pinvB = pinv(B)
r = np.dot(pinvB, sols.transpose([1, 0, 2]))
r = r[source_selection, :, :]
return r, source_selection
def _fit_tec_screen(station_names, source_names, pp, airmass, rr, weights, times,
height, order, r_0, beta, outQueue):
"""
Fits a screen to given TEC values using Karhunen-Lo`eve base vectors
Parameters
----------
station_names: array
Array of station names
source_names: array
Array of source names
pp: array
Array of piercepoint locations
airmass: array
Array of airmass values (note: not currently used)
rr: array
Array of TEC values to fit screen to
weights: array
Array of weights
times: array
Array of times
height: float
Height of screen (m)
order: int
Order of screen (i.e., number of KL base vectors to keep)
r_0: float
Scale size of phase fluctuations (m)
beta: float
Power-law index for phase structure function (5/3 => pure Kolmogorov
turbulence)
"""
import numpy as np
from pylab import kron, concatenate, pinv, norm, newaxis, find, amin, svd, eye
logging.info('Fitting screens...')
# Initialize arrays
N_stations = len(station_names)
N_sources = len(source_names)
N_times = len(times)
N_piercepoints = N_sources * N_stations
tec_fit_white_all = np.zeros((N_times, N_sources, N_stations))
tec_residual_all = np.zeros((N_times, N_sources, N_stations))
for k in range(N_times):
D = np.resize(pp[k, :, :], (N_piercepoints, N_piercepoints, 3))
D = np.transpose(D, (1, 0, 2)) - D
D2 = np.sum(D**2, axis=2)
C = -(D2 / r_0**2)**(beta / 2.0) / 2.0
pinvC = pinv(C, rcond=1e-3)
U, S, V = svd(C)
invU = pinv(np.dot(np.transpose(U[:, :order]), np.dot(weights[:, :, k], U[:, :order])), rcond=1e-3)
# Calculate screen
rr1 = np.dot(np.transpose(U[:, :order]), np.dot(weights[:, :, k], rr[:, k]))
tec_fit = np.dot(pinvC, np.dot(U[:, :order], np.dot(invU, rr1)))
tec_fit_white_all[k, :, :] = tec_fit.reshape((N_sources, N_stations))
residual = rr - np.dot(C, tec_fit)[:, newaxis]
tec_residual_all[k, :, :] = residual.reshape((N_sources, N_stations))
outQueue.put([tec_fit_white_all, tec_residual_all, times])
def _fit_phase_screen(station_names, source_names, pp, airmass, rr, weights, times,
height, order, r_0, beta, outQueue):
"""
Fits a screen to given phase values using Karhunen-Lo`eve base vectors
Parameters
----------
station_names: array
Array of station names
source_names: array
Array of source names
pp: array
Array of piercepoint locations
airmass: array
Array of airmass values (note: not currently used)
rr: array
Array of phase values to fit screen to
weights: array
Array of weights
times: array
Array of times
height: float
Height of screen (m)
order: int
Order of screen (i.e., number of KL base vectors to keep)
r_0: float
Scale size of phase fluctuations (m)
beta: float
Power-law index for phase structure function (5/3 => pure Kolmogorov
turbulence)
"""
import numpy as np
from pylab import kron, concatenate, pinv, norm, newaxis, find, amin, svd, eye
logging.info('Fitting screens...')
# Initialize arrays
N_stations = len(station_names)
N_sources = len(source_names)
N_times = len(times)
N_piercepoints = N_sources * N_stations
real_fit_white_all = np.zeros((N_times, N_sources, N_stations))
imag_fit_white_all = np.zeros((N_times, N_sources, N_stations))
phase_fit_white_all = np.zeros((N_times, N_sources, N_stations))
real_residual_all = np.zeros((N_times, N_sources, N_stations))
imag_residual_all = np.zeros((N_times, N_sources, N_stations))
phase_residual_all = np.zeros((N_times, N_sources, N_stations))
# Change phase to real/imag
rr_real = np.cos(rr)
rr_imag = np.sin(rr)
for k in range(N_times):
try:
D = np.resize(pp[k, :, :], (N_piercepoints, N_piercepoints, 3))
D = np.transpose(D, (1, 0, 2)) - D
D2 = np.sum(D**2, axis=2)
C = -(D2 / r_0**2)**(beta / 2.0) / 2.0
pinvC = pinv(C, rcond=1e-3)
U, S, V = svd(C)
invU = pinv(np.dot(np.transpose(U[:, :order]), np.dot(weights[:, :, k], U[:, :order])), rcond=1e-3)
# Calculate real screen
rr1 = np.dot(np.transpose(U[:, :order]), np.dot(weights[:, :, k], rr_real[:, k]))
real_fit = np.dot(pinvC, np.dot(U[:, :order], np.dot(invU, rr1)))
real_fit_white_all[k, :, :] = real_fit.reshape((N_sources, N_stations))
residual = rr_real - np.dot(C, real_fit)[:, newaxis]
real_residual_all[k, :, :] = residual.reshape((N_sources, N_stations))
# Calculate imag screen
rr1 = np.dot(np.transpose(U[:, :order]), np.dot(weights[:, :, k], rr_imag[:, k]))
imag_fit = np.dot(pinvC, np.dot(U[:, :order], np.dot(invU, rr1)))
imag_fit_white_all[k, :, :] = imag_fit.reshape((N_sources, N_stations))
residual = rr_imag - np.dot(C, imag_fit)[:, newaxis]
imag_residual_all[k, :, :] = residual.reshape((N_sources, N_stations))
# Calculate phase screen
phase_fit = np.dot(pinvC, np.arctan2(np.dot(C, imag_fit), np.dot(C, real_fit)))
phase_fit_white_all[k, :, :] = phase_fit.reshape((N_sources, N_stations))
residual = rr - np.dot(C, phase_fit)[:, newaxis]
phase_residual_all[k, :, :] = residual.reshape((N_sources, N_stations))
except:
# Set screen to zero if fit did not work
logging.debug('Screen fit failed for timeslot {}'.format(k))
real_fit_white_all[k, :, :] = np.zeros((N_sources, N_stations))
real_residual_all[k, :, :] = np.ones((N_sources, N_stations))
imag_fit_white_all[k, :, :] = np.zeros((N_sources, N_stations))
imag_residual_all[k, :, :] = np.ones((N_sources, N_stations))
phase_fit_white_all[k, :, :] = np.zeros((N_sources, N_stations))
phase_residual_all[k, :, :] = np.ones((N_sources, N_stations))
outQueue.put([real_fit_white_all, real_residual_all,
imag_fit_white_all, imag_residual_all,
phase_fit_white_all, phase_residual_all,
times])
def correction_light(I, method, show_light, mask=None):
"""Corrige la derive eclairement
:I: array_like ou iplimage
:method: 'polynomial' or 'frequency'
:show_light: option affiche correction (true|false)
:mask: array de zone non interet
:returns: iplimage 32bit
"""
from progress import *
import Tkinter
if type(I) == cv.iplimage:
if I.nChannels == 3:
if method == 'None':
I = RGB2L(I)
I = cv2array(I)[:, :, 0]
I = pymorph.hmin(I, 15, pymorph.sedisk(3))
I = array2cv(I)
cv.EqualizeHist(I, I)
return I
I = RGB2L(I)
I_32bit = cv.CreateImage(cv.GetSize(I), cv.IPL_DEPTH_32F, 1)
cv.ConvertScale(I, I_32bit, 3000.0, 0.0)
I = cv.CloneImage(I_32bit)
I = cv2array(I)[:, :, 0]
elif len(I.shape) == 3:
I = (I[:, :, 0] + I[:, :, 0] + I[:, :, 0])\
/ 3.0 # A modifier: non utiliser dans notre cas
elif method == 'None':
I = array2cv(I)
cv.EqualizeHist(I, I)
return I
I = np.log(I + 10 ** (-6))
(H, W) = np.shape(I)
I_out = I * 0 + 10 ** (-6)
if method == 'polynomial':
## I = M.A avec A coeff. du polynome
I_flat = I.flatten()
degree = 3
print("modification degree 3")
#degree du polynome
nb_coeff = (degree + 1) * (degree + 2) / 2 # nombre coefficient
[yy, xx] = np.meshgrid(np.arange(W, dtype=np.float64),
np.arange(H, dtype=np.float64))
if mask is not None:
xx[mask] = 0
yy[mask] = 0
# Creation de M
try:
M = np.zeros((H * W, nb_coeff), dtype=np.float64)
except MemoryError:
print MemoryError
return MemoryError
i, j = 0, 0 # i,j degree de x,y
#Bar progression
bar = Tkinter.Tk(className='Correcting Light...')
m = Meter(bar, relief='ridge', bd=3)
m.pack(fill='x')
m.set(0.0, 'Starting correction...')
for col in np.arange(nb_coeff):
M[:, col] = (xx.flatten() ** i) * (yy.flatten() ** j)
i += 1
m.set(0.5 * float(col) / (nb_coeff - 1))
if i + j == degree + 1:
i = 0
j += 1
# Resolution au sens des moindres carree: pseudo-inverse
try:
M = pl.pinv(M)
A = np.dot(M, I_flat)
except ValueError:
return ValueError
# Calcul de la surface
i, j = 0, 0
surface = np.zeros((H, W), dtype=np.float64)
for cmpt in np.arange(nb_coeff):
surface += A[cmpt] * (xx ** i) * (yy ** j) # forme quadratique
i += 1
m.set(0.5 + 0.5 * float(cmpt) / (nb_coeff - 1))
if i + j == degree + 1:
i = 0
j += 1
bar.destroy()
I_out = np.exp(I / surface)
light = surface
elif method == 'frequency':
Rx, Ry = 2, 2
# zero padding
N = [H, W]
filtre = np.zeros((N[1], N[0]))
centre_x = round(N[0] / 2)
centre_y = round(N[1] / 2)
print("FFT2D...")
I_fourier = pl.fftshift(pl.fft2(I, N))
# Gaussian filter
[xx, yy] = np.meshgrid(np.arange(N[0], dtype=np.float),
np.arange(N[1], dtype=np.float))
filtre = np.exp(-2 * ((xx - centre_x) ** 2 + (yy - centre_y) ** 2) /
(Rx ** 2 + Ry ** 2))
filtre = pl.transpose(filtre)
I_fourier = I_fourier * filtre
print("IFFT2D...")
I_out = (np.abs(pl.ifft2(pl.ifftshift(I_fourier), N)))[0:H, 0:W]
light = I_out
I_out = np.exp(I / I_out)
else:
light = I * 0
I_out = I
# Display Light
if show_light:
light = ((light - light.min()) * 3000.0 /
light.max()).astype('float32')
light = array2cv(light)
fig = pl.figure()
pl.imshow(light)
fig.show()
I_out = (I_out - I_out.min()) * 3000.0 / I_out.max()
I_out = I_out.astype('uint8')
#chapeau haut de forme
I_out = pymorph.hmin(I_out, 25, pymorph.sedisk(3))
#Conversion en iplimage et ajustement contraste
gr = array2cv(I_out)
cv.EqualizeHist(gr, gr)
return gr
def make_tec_screen_plots(pp, rr, tec_fit_white, tec_fit, station_positions,
source_names, times, height, order, beta_val, r_0, prefix = 'frame_',
remove_gradient=True):
"""Makes plots of TEC screens"""
import pylab
import numpy as np
import os
from operations.tecscreen import calc_piercepoint
import progressbar
root_dir = os.path.dirname(prefix)
if root_dir == '':
root_dir = './'
prestr = os.path.basename(prefix)
try:
os.makedirs(root_dir)
except OSError:
pass
N_stations = station_positions.shape[0]
N_sources = len(source_names)
N_times = len(times)
A = pylab.concatenate([pylab.kron(np.eye(N_sources),
np.ones((N_stations,1))), pylab.kron(np.ones((N_sources,1)),
np.eye(N_stations))], axis=1)
N_piercepoints = N_sources * N_stations
P = np.eye(N_piercepoints) - np.dot(np.dot(A, pylab.pinv(np.dot(A.T, A))), A.T)
x,y,z = station_positions[0, :]
east = np.array([-y, x, 0])
east = east / pylab.norm(east)
north = np.array([ -x, -y, (x*x + y*y)/z])
north = north / pylab.norm(north)
up = np.array([x,y,z])
up = up / pylab.norm(up)
T = pylab.concatenate([east[:, pylab.newaxis], north[:, pylab.newaxis]], axis=1)
pp1 = np.dot(pp[0, :, :], T)
lower = np.amin(pp1, axis=0)
upper = np.amax(pp1, axis=0)
extent = upper - lower
lower = lower - 0.05 * extent
upper = upper + 0.05 * extent
extent = upper - lower
N = 50
xr = np.arange(lower[0], upper[0], extent[0]/N)
yr = np.arange(lower[1], upper[1], extent[1]/N)
screen = np.zeros((N, N, N_times))
fitted_tec1 = tec_fit.transpose([0, 2, 1]).reshape(N_piercepoints, N_times) + np.dot(P, rr-tec_fit.transpose([0, 2, 1]).reshape(N_piercepoints, N_times))
logging.info('Calculating TEC screen images...')
pbar = progressbar.ProgressBar(maxval=N_times).start()
ipbar = 0
for k in range(N_times):
f = tec_fit_white[:, k, :]
for i, x in enumerate(xr[0: N]):
for j, y in enumerate(yr[0: N]):
p = calc_piercepoint(np.dot(np.array([x, y]), np.array([east, north])), up, height)
d2 = np.sum(np.square(pp[k, :, :] - p[0]), axis=1)
c = -(d2 / ( r_0**2 ) )**( beta_val / 2.0 ) / 2.0
screen[j, i, k] = np.dot(c, f.reshape(N_piercepoints))
# Fit and remove a gradient
if remove_gradient:
xs, ys = np.indices(screen.shape[0:2])
zs = screen[:, :, k]
XYZ = []
for xf, yf in zip(xs.flatten().tolist(), ys.flatten().tolist()):
XYZ.append([xf, yf, zs[xf, yf]])
XYZ = np.array(XYZ)
a, b, c = fitPLaneLTSQ(XYZ)
grad_plane = a * xs + b * ys + c
screen[:, :, k] = screen[:, :, k] - grad_plane
for t in range(fitted_tec1.shape[0]):
xs_pt = np.where(np.array(xr) > pp1[t, 0])[0][0]
ys_pt = np.where(np.array(yr) > pp1[t, 1])[0][0]
grad_plane_pt = a * ys_pt + b * xs_pt + c
fitted_tec1[t, k] = fitted_tec1[t, k] - grad_plane_pt
pbar.update(ipbar)
ipbar += 1
pbar.finish()
vmin = np.min([np.amin(screen), np.amin(fitted_tec1)])
vmax = np.max([np.amax(screen), np.amax(fitted_tec1)])
logging.info('Plotting TEC screens...')
fig1 = pylab.figure(figsize = (7, 7))
pbar = progressbar.ProgressBar(maxval=N_times).start()
ipbar = 0
for k in range(N_times):
pylab.clf()
im = pylab.imshow(screen[:, :, k],
cmap = pylab.cm.jet,
origin = 'lower',
interpolation = 'nearest',
extent = (xr[0], xr[-1], yr[0], yr[-1]), vmin = vmin, vmax = vmax)
sm = pylab.cm.ScalarMappable(cmap = pylab.cm.jet,
norm = pylab.normalize(vmin = vmin, vmax=vmax))
sm._A = []
pylab.title(str(i))
cbar = pylab.colorbar()
cbar.set_label('TECU', rotation=270)
x = []
y = []
s = []
c = []
for j in range(fitted_tec1.shape[0]):
x.append(pp1[j,0])
y.append(pp1[j,1])
xs = np.where(np.array(xr) > pp1[j,0])[0][0]
ys = np.where(np.array(yr) > pp1[j,1])[0][0]
s.append(max(2400*abs(fitted_tec1[j, k] - screen[ys, xs, k]), 10))
c.append(sm.to_rgba(fitted_tec1[j, k]))
pylab.scatter(x, y, s=s, c=c)
labels = source_names
for label, xl, yl in zip(labels, x[0::N_stations], y[0::N_stations]):
pylab.annotate(
label,
xy = (xl, yl), xytext = (-20, 20),
textcoords = 'offset points', ha = 'right', va = 'bottom',
bbox = dict(boxstyle = 'round,pad=0.5', fc = 'gray', alpha = 0.5),
arrowprops = dict(arrowstyle = '->', connectionstyle = 'arc3,rad=0'))
pylab.title('Time {0}'.format(k))
pylab.xlim(xr[-1], xr[0])
pylab.ylim(yr[0], yr[-1])
pylab.xlabel('Projected Distance (m)')
pylab.ylabel('Projected Distance (m)')
pylab.savefig(root_dir+'/'+prestr+'frame%0.3i.png' % k)
pbar.update(ipbar)
ipbar += 1
pbar.finish()
pylab.close(fig1)
def fit_tec_per_source_pair(phases, flags, mask, freqs, init_sols=None,
init_sols_per_pair=False, propagate=False, nband_min=2):
"""Fits TEC values to phase solutions per source pair
Returns TEC solutions as array of shape (N_sources, N_times, N_stations)
Keyword arguments:
phases -- array of phase solutions
flags -- phase solution flags (0 = use, 1 = flagged)
mask -- mask for sources and frequencies (0 = ignore, 1 = use)
freqs -- array of frequencies
init_sols -- solutions to use to initialize the fits
init_sols_per_pair -- init_sols are per source pair (not per source)
propagate -- propagate solutions from previous solution
nband_min -- min number of bands for a source to be used
"""
from pylab import pinv, newaxis, find
import numpy as np
from lofar.expion import baselinefitting
try:
import progressbar
except ImportError:
import losoto.progressbar as progressbar
sols_list = []
eq_list = []
vars_list = []
var_eq_list = []
N_sources = phases.shape[0]
N_stations = phases.shape[1]
N_times = phases.shape[3]
N_pairs = 0
for i in xrange(N_sources):
for j in xrange(i):
subband_selection = find(mask[i, :] * mask[j, :])
if len(subband_selection) < nband_min:
continue
N_pairs += 1
if init_sols is None and not init_sols_per_pair:
init_sols = np.zeros((N_sources, N_times, N_stations), dtype = np.float)
elif init_sols is None and init_sols_per_pair:
init_sols = np.zeros((N_pairs, N_times, N_stations), dtype = np.float)
source_pairs = []
k = 0
ipbar = 0
logging.info('Fitting TEC values...')
pbar = progressbar.ProgressBar(maxval=N_pairs*N_times).start()
for i in xrange(N_sources):
for j in xrange(i):
subband_selection = find(mask[i, :] * mask[j, :])
if len(subband_selection) < nband_min:
continue
source_pairs.append((i, j))
p = phases[i, :, subband_selection, :] - phases[j, :, subband_selection, :]
A = np.zeros((len(subband_selection), 1))
A[:, 0] = 8.44797245e9/freqs[subband_selection]
flags_source_pair = flags[i, :, subband_selection, :] * flags[j, :,
subband_selection, :]
constant_parms = np.zeros((1, N_stations), dtype = np.bool)
sols = np.zeros((N_times, N_stations), dtype = np.float)
if init_sols_per_pair:
p_0 = init_sols[k, 0, :][newaxis, :]
else:
p_0 = (init_sols[i, 0, :] - init_sols[j, 0, :])[newaxis, :]
for t_idx in xrange(N_times):
x = p[:, :, t_idx].copy()
f = flags_source_pair[:, :, t_idx].copy()
if not propagate:
if init_sols_per_pair:
p_0 = init_sols[k, t_idx, :][newaxis, :]
else:
p_0 = (init_sols[i, t_idx, :] - init_sols[j, 0, :])[newaxis, :]
sol = baselinefitting.fit(x, A, p_0, f, constant_parms)
if propagate:
p_0 = sol.copy()
sols[t_idx, :] = sol[0, :]
ipbar += 1
pbar.update(ipbar)
sols = sols[:, :] - np.mean(sols[:, :], axis=1)[:, newaxis]
weight = len(subband_selection)
sols_list.append(sols*weight)
eq = np.zeros(N_sources)
eq[i] = weight
eq[j] = -weight
eq_list.append(eq)
k += 1
pbar.finish()
sols = np.array(sols_list)
B = np.array(eq_list)
source_selection = find(np.sum(abs(B), axis=0))
N_sources = len(source_selection)
if N_sources == 0:
logging.error('All sources have fewer than the required minimum number of bands.')
return None, None
pinvB = pinv(B)
r = np.dot(pinvB, sols.transpose([1, 0, 2]))
r = r[source_selection, :, :]
return r, source_selection
def _fit_tec_screen(station_names, source_names, pp, airmass, rr, weights,
times, height, order, r_0, beta, outQueue):
"""
Fits a screen to given TEC values using Karhunen-Lo`eve base vectors
Parameters
----------
station_names: array
Array of station names
source_names: array
Array of source names
pp: array
Array of piercepoint locations
airmass: array
Array of airmass values (note: not currently used)
rr: array
Array of TEC values to fit screen to
weights: array
Array of weights
times: array
Array of times
height: float
Height of screen (m)
order: int
Order of screen (i.e., number of KL base vectors to keep)
r_0: float
Scale size of phase fluctuations (m)
beta: float
Power-law index for phase structure function (5/3 => pure Kolmogorov
turbulence)
"""
import numpy as np
from pylab import kron, concatenate, pinv, norm, newaxis, find, amin, svd, eye
logging.info('Fitting screens...')
# Initialize arrays
N_stations = len(station_names)
N_sources = len(source_names)
N_times = len(times)
N_piercepoints = N_sources * N_stations
tec_fit_white_all = np.zeros((N_times, N_sources, N_stations))
tec_residual_all = np.zeros((N_times, N_sources, N_stations))
for k in range(N_times):
D = np.resize(pp[k, :, :], (N_piercepoints, N_piercepoints, 3))
D = np.transpose(D, (1, 0, 2)) - D
D2 = np.sum(D**2, axis=2)
C = -(D2 / r_0**2)**(beta / 2.0) / 2.0
pinvC = pinv(C, rcond=1e-3)
U, S, V = svd(C)
invU = pinv(np.dot(np.transpose(U[:, :order]),
np.dot(weights[:, :, k], U[:, :order])),
rcond=1e-3)
# Calculate screen
rr1 = np.dot(np.transpose(U[:, :order]),
np.dot(weights[:, :, k], rr[:, k]))
tec_fit = np.dot(pinvC, np.dot(U[:, :order], np.dot(invU, rr1)))
tec_fit_white_all[k, :, :] = tec_fit.reshape((N_sources, N_stations))
residual = rr - np.dot(C, tec_fit)[:, newaxis]
tec_residual_all[k, :, :] = residual.reshape((N_sources, N_stations))
outQueue.put([tec_fit_white_all, tec_residual_all, times])
def getControlMaps(state_r, state_l, dataset, conf=None, indices=None):
"""
This function creates the linear mappings that are required for creating a controlled SLIP:
* parameter prediction schemes
These are mappings from [CoM state, additional states] -> [SLIP parameters],
which tell the system how to update the SLIP parameters each step
* Non-SLIP state propagators
These are mappings from [CoM state, additional states] -> [additional states],
which tell the system how the additional states (not part of original SLIP)
evolve from apex to apex (discrete dynamics).
As these mappings are affine mappings, the corresponding lifts (constants) are also given.
This implies that a periodic SLIP solution is computed.
This function uses bootstrap.
*NOTE* Do *not* detrend the input - use "original" data!
*NOTE* It is assumed that the first 3 dimensions of the data contain the CoM state at apex!
:args:
state_r (n-by-d array): the full state of the system at right apices.
First three dimensions are considered as state of the CoM
[height, velocity in horizontal plane (2x)]
state_l (n-by-d array): the full state of the system at left apices.
*NOTE* it is assumed that a left apex follows a right apex, i.e. state_r[0, :] is
before state_l[0, :]
dataset: a dataset obtained from build_dataset(). Essentially, this is an object with
.all_param_r [n-by-5 array]
.all_param_l [n-by-5 array]
.yminL [list, n elements]
.yminR [list, n elements]
.TR [list, n elements]
.TL [list, n elements]
.masses [list, n elements]
conf: a mutils.io.saveable object, containing detrending info (int
.dt_window and bool .dt_medfilter fields)
indices (list or 1d array of int): indices to use for regression. If
NONE, use bootstrap and average over matrices.
:returns:
(ctrl_r, ctrl_l), (prop_r, prop_l), (ref_state_r, ref_state_l, ref_param_r, ref_param_l,
ref_addDict)
Tuples containing the (1) parameter update maps, (2) state propagator maps,
(3) reference states and parameters and additional SLIP
"""
if conf == None:
conf = mio.saveable()
conf.dt_window = 30
conf.dt_medfilter = False
if indices is not None:
indices = array(indices).squeeze()
d = dataset # shortcut
addDict = {'m': mean(d.masses), 'g': -9.81}
#def getPeriodicOrbit2(ICr, Tr, yminr, ICl, Tl, yminl, m, startParams=[14000.,
#[ICp_r, Pp_r, dE_r], [ICp_l, Pp_l, dE_l] =
ICp_r = mean(d.all_IC_r, axis=0)
ICp_l = mean(d.all_IC_l, axis=0)
TR = mean(vstack(d.TR))
TL = mean(vstack(d.TL))
yminr = mean(vstack(d.yminR))
yminl = mean(vstack(d.yminL))
Pp_r, Pp_l = getPeriodicOrbit2(ICp_r,
TR,
yminr,
ICp_l,
TL,
yminl,
mean(d.masses),
startParams=mean(vstack(d.all_param_r),
axis=0)[:5])
# change last element to be energy input - this is consistent with the format used in the rest of the code
if False: # obsolete code
Pp_r = array(Pp_r)
Pp_l = array(Pp_l)
Pp_r[4] = dE_r
Pp_r = Pp_r[:5]
Pp_l[4] = dE_l
Pp_l = Pp_l[:5]
# now: detrend data
# non-SLIP state
dt_nss_r = fda.dt_movingavg(state_r[:, 3:], conf.dt_window,
conf.dt_medfilter)
dt_nss_l = fda.dt_movingavg(state_l[:, 3:], conf.dt_window,
conf.dt_medfilter)
# full state
dt_fs_r = fda.dt_movingavg(state_r, conf.dt_window, conf.dt_medfilter)
dt_fs_l = fda.dt_movingavg(state_l, conf.dt_window, conf.dt_medfilter)
# SLIP parameters
dt_pl = fda.dt_movingavg(d.all_param_l, conf.dt_window, conf.dt_medfilter)
dt_pr = fda.dt_movingavg(d.all_param_r, conf.dt_window, conf.dt_medfilter)
# compute non-SLIP state prediction maps (propagators)
if indices is None:
_, all_prop_r, _ = fda.fitData(dt_fs_r,
dt_nss_l,
nps=1,
nrep=100,
sections=[
0,
],
rcond=1e-8)
_, all_prop_l, _ = fda.fitData(dt_fs_l[:-1, :],
dt_nss_r[1:, :],
nps=1,
nrep=100,
sections=[
0,
],
rcond=1e-8)
prop_r = fda.meanMat(all_prop_r)
prop_l = fda.meanMat(all_prop_l)
else:
prop_r = dot(dt_nss_l[indices, :].T,
pinv(dt_fs_r[indices, :].T, rcond=1e-8))
prop_l = dot(dt_nss_r[indices[:-1] + 1, :].T,
pinv(dt_fs_l[indices[:-1], :].T, rcond=1e-8))
# compute parameter prediction maps
if indices is None:
_, all_ctrl_r, _ = fda.fitData(dt_fs_r,
dt_pr,
nps=1,
nrep=100,
sections=[
0,
],
rcond=1e-8)
_, all_ctrl_l, _ = fda.fitData(dt_fs_l,
dt_pl,
nps=1,
nrep=100,
sections=[
0,
],
rcond=1e-8)
ctrl_r = fda.meanMat(all_ctrl_r)
ctrl_l = fda.meanMat(all_ctrl_l)
else:
ctrl_r = dot(dt_pr[indices, :].T,
pinv(dt_fs_r[indices, :].T, rcond=1e-8))
ctrl_l = dot(dt_pl[indices, :].T,
pinv(dt_fs_l[indices, :].T, rcond=1e-8))
return (ctrl_r, ctrl_l), (prop_r, prop_l), (ICp_r, ICp_l, Pp_r, Pp_l,
addDict)
def getControlMaps(state_r, state_l, dataset, conf=None, indices=None):
"""
This function creates the linear mappings that are required for creating a controlled SLIP:
* parameter prediction schemes
These are mappings from [CoM state, additional states] -> [SLIP parameters],
which tell the system how to update the SLIP parameters each step
* Non-SLIP state propagators
These are mappings from [CoM state, additional states] -> [additional states],
which tell the system how the additional states (not part of original SLIP)
evolve from apex to apex (discrete dynamics).
As these mappings are affine mappings, the corresponding lifts (constants) are also given.
This implies that a periodic SLIP solution is computed.
This function uses bootstrap.
*NOTE* Do *not* detrend the input - use "original" data!
*NOTE* It is assumed that the first 3 dimensions of the data contain the CoM state at apex!
:args:
state_r (n-by-d array): the full state of the system at right apices.
First three dimensions are considered as state of the CoM
[height, velocity in horizontal plane (2x)]
state_l (n-by-d array): the full state of the system at left apices.
*NOTE* it is assumed that a left apex follows a right apex, i.e. state_r[0, :] is
before state_l[0, :]
dataset: a dataset obtained from build_dataset(). Essentially, this is an object with
.all_param_r [n-by-5 array]
.all_param_l [n-by-5 array]
.yminL [list, n elements]
.yminR [list, n elements]
.TR [list, n elements]
.TL [list, n elements]
.masses [list, n elements]
conf: a mutils.io.saveable object, containing detrending info (int
.dt_window and bool .dt_medfilter fields)
indices (list or 1d array of int): indices to use for regression. If
NONE, use bootstrap and average over matrices.
:returns:
(ctrl_r, ctrl_l), (prop_r, prop_l), (ref_state_r, ref_state_l, ref_param_r, ref_param_l,
ref_addDict)
Tuples containing the (1) parameter update maps, (2) state propagator maps,
(3) reference states and parameters and additional SLIP
"""
if conf == None:
conf = mio.saveable()
conf.dt_window=30
conf.dt_medfilter=False
if indices is not None:
indices = array(indices).squeeze()
d = dataset # shortcut
addDict = { 'm' : mean(d.masses), 'g' : -9.81 }
#def getPeriodicOrbit2(ICr, Tr, yminr, ICl, Tl, yminl, m, startParams=[14000.,
#[ICp_r, Pp_r, dE_r], [ICp_l, Pp_l, dE_l] =
ICp_r = mean(d.all_IC_r, axis=0)
ICp_l = mean(d.all_IC_l, axis=0)
TR = mean(vstack(d.TR))
TL = mean(vstack(d.TL))
yminr = mean(vstack(d.yminR))
yminl = mean(vstack(d.yminL))
Pp_r, Pp_l = getPeriodicOrbit2(ICp_r, TR, yminr, ICp_l, TL, yminl,
mean(d.masses), startParams=mean(vstack(d.all_param_r), axis=0)[:5])
# change last element to be energy input - this is consistent with the format used in the rest of the code
if False: # obsolete code
Pp_r = array(Pp_r)
Pp_l = array(Pp_l)
Pp_r[4] = dE_r
Pp_r = Pp_r[:5]
Pp_l[4] = dE_l
Pp_l = Pp_l[:5]
# now: detrend data
# non-SLIP state
dt_nss_r = fda.dt_movingavg(state_r[:, 3:], conf.dt_window,
conf.dt_medfilter)
dt_nss_l = fda.dt_movingavg(state_l[:, 3:],conf.dt_window,
conf.dt_medfilter)
# full state
dt_fs_r = fda.dt_movingavg(state_r, conf.dt_window, conf.dt_medfilter)
dt_fs_l = fda.dt_movingavg(state_l, conf.dt_window, conf.dt_medfilter)
# SLIP parameters
dt_pl = fda.dt_movingavg(d.all_param_l, conf.dt_window, conf.dt_medfilter)
dt_pr = fda.dt_movingavg(d.all_param_r, conf.dt_window, conf.dt_medfilter)
# compute non-SLIP state prediction maps (propagators)
if indices is None:
_, all_prop_r, _ = fda.fitData(dt_fs_r, dt_nss_l, nps=1, nrep=100,
sections=[0,], rcond=1e-8)
_, all_prop_l, _ = fda.fitData(dt_fs_l[:-1, :], dt_nss_r[1:, :], nps=1,
nrep=100, sections=[0,], rcond=1e-8)
prop_r = fda.meanMat(all_prop_r)
prop_l = fda.meanMat(all_prop_l)
else:
prop_r = dot(dt_nss_l[indices, :].T, pinv(dt_fs_r[indices, :].T, rcond=1e-8))
prop_l = dot(dt_nss_r[indices[:-1] + 1, :].T, pinv(dt_fs_l[indices[:-1], :].T, rcond=1e-8))
# compute parameter prediction maps
if indices is None:
_, all_ctrl_r, _ = fda.fitData(dt_fs_r, dt_pr, nps=1, nrep=100, sections=[0,], rcond=1e-8)
_, all_ctrl_l, _ = fda.fitData(dt_fs_l, dt_pl, nps=1, nrep=100, sections=[0,], rcond=1e-8)
ctrl_r = fda.meanMat(all_ctrl_r)
ctrl_l = fda.meanMat(all_ctrl_l)
else:
ctrl_r = dot(dt_pr[indices, :].T, pinv(dt_fs_r[indices, :].T, rcond=1e-8))
ctrl_l = dot(dt_pl[indices, :].T, pinv(dt_fs_l[indices, :].T, rcond=1e-8))
return (ctrl_r, ctrl_l), (prop_r, prop_l), (ICp_r, ICp_l, Pp_r, Pp_l, addDict)
def fit_screen_to_tec(station_names, source_names, pp, airmass, rr, times,
height, order, r_0, beta):
"""
Fits a screen to given TEC values using Karhunen-Lo`eve base vectors
Keyword arguments:
station_names -- array of station names
source_names -- array of source names
pp -- array of piercepoint locations
airmass -- array of airmass values
rr -- array of TEC solutions
times -- array of times
height -- height of screen (m)
order -- order of screen (i.e., number of KL base vectors to keep)
r_0 -- scale size of phase fluctuations (m)
beta -- power-law index for phase structure function (5/3 =>
pure Kolmogorov turbulence)
"""
import numpy as np
from pylab import kron, concatenate, pinv, norm, newaxis, find, amin, svd, eye
try:
import progressbar
except ImportError:
import losoto.progressbar as progressbar
logging.info('Fitting screens to TEC values...')
N_stations = len(station_names)
N_sources = len(source_names)
N_times = len(times)
tec_fit_all = np.zeros((N_times, N_sources, N_stations))
residual_all = np.zeros((N_times, N_sources, N_stations))
A = concatenate([kron(eye(N_sources), np.ones((N_stations, 1))),
kron(np.ones((N_sources, 1)), eye(N_stations))], axis=1)
N_piercepoints = N_sources * N_stations
P = eye(N_piercepoints) - np.dot(np.dot(A, pinv(np.dot(A.T, A))), A.T)
pbar = progressbar.ProgressBar(maxval=N_times).start()
ipbar = 0
for k in range(N_times):
try:
D = np.resize(pp[k, :, :], (N_piercepoints, N_piercepoints, 3))
D = np.transpose(D, (1, 0, 2)) - D
D2 = np.sum(D**2, axis=2)
C = -(D2 / r_0**2)**(beta / 2.0) / 2.0
P1 = eye(N_piercepoints) - np.ones((N_piercepoints, N_piercepoints)) / N_piercepoints
C1 = np.dot(np.dot(P1, C), P1)
U, S, V = svd(C1)
B = np.dot(P, np.dot(np.diag(airmass[k, :]), U[:, :order]))
pinvB = pinv(B, rcond=1e-3)
rr1 = np.dot(P, rr[:, k])
tec_fit = np.dot(U[:, :order], np.dot(pinvB, rr1))
tec_fit_all[k, :, :] = tec_fit.reshape((N_sources, N_stations))
residual = rr1 - np.dot(P, tec_fit)
residual_all[k, :, :] = residual.reshape((N_sources, N_stations))
except:
# Set screen to zero if fit did not work
logging.debug('Tecscreen fit failed for timeslot {0}'.format(k))
tec_fit_all[k, :, :] = np.zeros((N_sources, N_stations))
residual_all[k, :, :] = np.ones((N_sources, N_stations))
pbar.update(ipbar)
ipbar += 1
pbar.finish()
return tec_fit_all, residual_all
known_columns.append(c)
dG0_f[c, 0] = temp
except Exception:
continue
unknown_columns = sorted(list(set(range(len(all_cids))).difference(known_columns))) # find all the indices of columns not in known_columns
unknown_rows = find(isnan(dG0_r))
known_rows = sorted(list(set(range(len(reactions))).difference(unknown_rows))) # find all the indices of rows with measured dG0_r
S_measured = S[known_rows, :]
b = dG0_r[known_rows] - dot(S_measured[:, known_columns], dG0_f[known_columns])
S_red = S_measured[:, unknown_columns]
print "Formation energies from from GC and from linear regression: "
# linear regression to solve the missing formation energies (S*x = b)
inv_corr_mat = pinv(dot(S_red.T, S_red))
x = dot(dot(inv_corr_mat, S_red.T), b)
for c in xrange(len(unknown_columns)):
dG0_f[unknown_columns[c], 0] = x[c, 0]
csv_out = csv.writer(open('../res/acetogens_compounds.csv', 'w'))
for c in xrange(len(all_cids)):
if (c in known_columns):
csv_out.writerow(['C%05d' % all_cids[c], '%8.1f' % dG0_f[c, 0], 'Group Contribution'])
else:
csv_out.writerow(['C%05d' % all_cids[c], '%8.1f' % dG0_f[c, 0], 'Calculated'])
#figure()
#plot(b, dot(S_red, dG0_f[unknown_columns]), '.')
figure()
plot(dG0_r[known_rows], dot(S[known_rows, :], dG0_f), '.')
def make_tec_screen_plots(pp,
tec_screen,
residuals,
station_positions,
source_names,
times,
height,
order,
beta_val,
r_0,
prefix='frame_',
remove_gradient=True,
show_source_names=False,
min_tec=None,
max_tec=None):
"""Makes plots of TEC screens
Keyword arguments:
pp -- array of piercepoint locations
tec_screen -- array of TEC screen values at the piercepoints
residuals -- array of TEC screen residuals at the piercepoints
source_names -- array of source names
times -- array of times
height -- height of screen (m)
order -- order of screen (e.g., number of KL base vectors to keep)
r_0 -- scale size of phase fluctuations (m)
beta_val -- power-law index for phase structure function (5/3 =>
pure Kolmogorov turbulence)
prefix -- prefix for output file names
remove_gradient -- fit and remove a gradient from each screen
show_source_names -- label sources on screen plots
min_tec -- minimum TEC value for plot range
max_tec -- maximum TEC value for plot range
"""
from pylab import kron, concatenate, pinv, norm, newaxis, normalize
import matplotlib.pyplot as plt
from mpl_toolkits.axes_grid1.inset_locator import inset_axes
import numpy as np
import os
from operations.tecscreen import calc_piercepoint
import progressbar
root_dir = os.path.dirname(prefix)
if root_dir == '':
root_dir = './'
prestr = os.path.basename(prefix) + 'screen_'
try:
os.makedirs(root_dir)
except OSError:
pass
N_stations = station_positions.shape[0]
N_sources = len(source_names)
N_times = len(times)
A = concatenate([
kron(np.eye(N_sources), np.ones((N_stations, 1))),
kron(np.ones((N_sources, 1)), np.eye(N_stations))
],
axis=1)
N_piercepoints = N_sources * N_stations
P = np.eye(N_piercepoints) - np.dot(np.dot(A, pinv(np.dot(A.T, A))), A.T)
x, y, z = station_positions[0, :]
east = np.array([-y, x, 0])
east = east / norm(east)
north = np.array([-x, -y, (x * x + y * y) / z])
north = north / norm(north)
up = np.array([x, y, z])
up = up / norm(up)
T = concatenate([east[:, newaxis], north[:, newaxis]], axis=1)
pp1 = np.dot(pp[0, :, :], T)
lower = np.amin(pp1, axis=0)
upper = np.amax(pp1, axis=0)
extent = upper - lower
lower = lower - 0.05 * extent
upper = upper + 0.05 * extent
extent = upper - lower
Nx = 25
Ny = int(extent[1] / extent[0] * np.float(Nx))
xr = np.arange(lower[0], upper[0], extent[0] / Nx)
yr = np.arange(lower[1], upper[1], extent[1] / Ny)
screen = np.zeros((Nx, Ny, N_times))
gradient = np.zeros((Nx, Ny, N_times))
residuals = residuals.transpose([0, 2, 1]).reshape(N_piercepoints, N_times)
fitted_tec1 = tec_screen.transpose([0, 2, 1]).reshape(
N_piercepoints, N_times) + residuals
logging.info('Calculating TEC screen images...')
pbar = progressbar.ProgressBar(maxval=N_times).start()
ipbar = 0
for k in range(N_times):
D = np.resize(pp[k, :, :], (N_piercepoints, N_piercepoints, 3))
D = np.transpose(D, (1, 0, 2)) - D
D2 = np.sum(D**2, axis=2)
C = -(D2 / r_0**2)**(beta_val / 2.0) / 2.0
f = np.dot(pinv(C), tec_screen[:, k, :].reshape(N_piercepoints))
for i, x in enumerate(xr[0:Nx]):
for j, y in enumerate(yr[0:Ny]):
p = calc_piercepoint(
np.dot(np.array([x, y]), np.array([east, north])), up,
height)
d2 = np.sum(np.square(pp[k, :, :] - p[0]), axis=1)
c = -(d2 / (r_0**2))**(beta_val / 2.0) / 2.0
screen[i, j, k] = np.dot(c, f)
# Fit and remove a gradient.
# Plot gradient in lower-left corner with its own color bar?
if remove_gradient:
xs, ys = np.indices(screen.shape[0:2])
zs = screen[:, :, k]
XYZ = []
for xf, yf in zip(xs.flatten().tolist(), ys.flatten().tolist()):
XYZ.append([xf, yf, zs[xf, yf]])
XYZ = np.array(XYZ)
a, b, c = fitPLaneLTSQ(XYZ)
grad_plane = a * xs + b * ys + c
gradient[:, :, k] = grad_plane
screen[:, :, k] = screen[:, :, k] - grad_plane
screen[:, :, k] = screen[:, :, k] - np.mean(screen[:, :, k])
for t in range(fitted_tec1.shape[0]):
xs_pt = np.where(np.array(xr) > pp1[t, 0])[0][0]
ys_pt = np.where(np.array(yr) > pp1[t, 1])[0][0]
grad_plane_pt = a * xs_pt + b * ys_pt + c
fitted_tec1[t, k] = fitted_tec1[t, k] - grad_plane_pt
fitted_tec1[:, k] = fitted_tec1[:, k] - np.mean(fitted_tec1[:, k])
pbar.update(ipbar)
ipbar += 1
pbar.finish()
if min_tec is None:
vmin = np.min([np.amin(screen), np.amin(fitted_tec1)])
else:
vmin = min_tec
if max_tec is None:
vmax = np.max([np.amax(screen), np.amax(fitted_tec1)])
else:
vmax = max_tec
logging.info('Plotting TEC screens...')
fig, ax = plt.subplots(figsize=[7, 7])
pbar = progressbar.ProgressBar(maxval=N_times).start()
ipbar = 0
for k in range(N_times):
plt.clf()
im = plt.imshow(screen.transpose([1, 0, 2])[:, :, k],
cmap=plt.cm.jet,
origin='lower',
interpolation='nearest',
extent=(xr[0] / 1000.0, xr[-1] / 1000.0,
yr[0] / 1000.0, yr[-1] / 1000.0),
vmin=vmin,
vmax=vmax)
sm = plt.cm.ScalarMappable(cmap=plt.cm.jet,
norm=normalize(vmin=vmin, vmax=vmax))
sm._A = []
cbar = plt.colorbar(im)
cbar.set_label('TECU', rotation=270)
x = []
y = []
s = []
c = []
for j in range(fitted_tec1.shape[0]):
x.append(pp1[j, 0] / 1000.0)
y.append(pp1[j, 1] / 1000.0)
xs = np.where(np.array(xr) > pp1[j, 0])[0][0]
ys = np.where(np.array(yr) > pp1[j, 1])[0][0]
fit_screen_diff = abs(fitted_tec1[j, k] - screen[xs, ys, k])
s.append(max(20 * fit_screen_diff / 0.01, 10))
c.append(sm.to_rgba(fitted_tec1[j, k]))
plt.scatter(x, y, s=s, c=c)
if show_source_names:
labels = source_names
for label, xl, yl in zip(labels, x[0::N_stations],
y[0::N_stations]):
plt.annotate(label,
xy=(xl, yl),
xytext=(-2, 2),
textcoords='offset points',
ha='right',
va='bottom')
plt.title('Screen {0}'.format(k))
plt.xlim(xr[-1] / 1000.0, xr[0] / 1000.0)
plt.ylim(yr[0] / 1000.0, yr[-1] / 1000.0)
plt.xlabel('Projected Distance along RA (km)')
plt.ylabel('Projected Distance along Dec (km)')
if remove_gradient:
axins = inset_axes(ax, width="15%", height="10%", loc=2)
axins.imshow(gradient.transpose([1, 0, 2])[:, :, k],
cmap=plt.cm.jet,
origin='lower',
interpolation='nearest',
extent=(xr[0] / 1000.0, xr[-1] / 1000.0,
yr[0] / 1000.0, yr[-1] / 1000.0),
vmin=vmin,
vmax=vmax)
plt.xticks(visible=False)
plt.yticks(visible=False)
axins.set_xlim(xr[-1] / 1000.0, xr[0] / 1000.0)
axins.set_ylim(yr[0] / 1000.0, yr[-1] / 1000.0)
plt.savefig(root_dir + '/' + prestr + 'frame%0.3i.png' % k)
pbar.update(ipbar)
ipbar += 1
pbar.finish()
plt.close(fig)
def make_tec_screen_plots(pp, tec_screen, residuals, station_positions,
source_names, times, height, order, beta_val, r_0, prefix = 'frame_',
remove_gradient=True, show_source_names=False, min_tec=None, max_tec=None):
"""Makes plots of TEC screens
Keyword arguments:
pp -- array of piercepoint locations
tec_screen -- array of TEC screen values at the piercepoints
residuals -- array of TEC screen residuals at the piercepoints
source_names -- array of source names
times -- array of times
height -- height of screen (m)
order -- order of screen (e.g., number of KL base vectors to keep)
r_0 -- scale size of phase fluctuations (m)
beta_val -- power-law index for phase structure function (5/3 =>
pure Kolmogorov turbulence)
prefix -- prefix for output file names
remove_gradient -- fit and remove a gradient from each screen
show_source_names -- label sources on screen plots
min_tec -- minimum TEC value for plot range
max_tec -- maximum TEC value for plot range
"""
from pylab import kron, concatenate, pinv, norm, newaxis, normalize
import matplotlib.pyplot as plt
from mpl_toolkits.axes_grid1.inset_locator import inset_axes
import numpy as np
import os
from operations.tecscreen import calc_piercepoint
import progressbar
root_dir = os.path.dirname(prefix)
if root_dir == '':
root_dir = './'
prestr = os.path.basename(prefix) + 'screen_'
try:
os.makedirs(root_dir)
except OSError:
pass
N_stations = station_positions.shape[0]
N_sources = len(source_names)
N_times = len(times)
A = concatenate([kron(np.eye(N_sources),
np.ones((N_stations,1))), kron(np.ones((N_sources,1)),
np.eye(N_stations))], axis=1)
N_piercepoints = N_sources * N_stations
P = np.eye(N_piercepoints) - np.dot(np.dot(A, pinv(np.dot(A.T, A))), A.T)
x, y, z = station_positions[0, :]
east = np.array([-y, x, 0])
east = east / norm(east)
north = np.array([ -x, -y, (x*x + y*y)/z])
north = north / norm(north)
up = np.array([x ,y, z])
up = up / norm(up)
T = concatenate([east[:, newaxis], north[:, newaxis]], axis=1)
pp1 = np.dot(pp[0, :, :], T)
lower = np.amin(pp1, axis=0)
upper = np.amax(pp1, axis=0)
extent = upper - lower
lower = lower - 0.05 * extent
upper = upper + 0.05 * extent
extent = upper - lower
Nx = 25
Ny = int(extent[1] / extent[0] * np.float(Nx))
xr = np.arange(lower[0], upper[0], extent[0]/Nx)
yr = np.arange(lower[1], upper[1], extent[1]/Ny)
screen = np.zeros((Nx, Ny, N_times))
gradient = np.zeros((Nx, Ny, N_times))
residuals = residuals.transpose([0, 2, 1]).reshape(N_piercepoints, N_times)
fitted_tec1 = tec_screen.transpose([0, 2, 1]).reshape(N_piercepoints, N_times) + residuals
logging.info('Calculating TEC screen images...')
pbar = progressbar.ProgressBar(maxval=N_times).start()
ipbar = 0
for k in range(N_times):
D = np.resize(pp[k, :, :], (N_piercepoints, N_piercepoints, 3))
D = np.transpose(D, (1, 0, 2)) - D
D2 = np.sum(D**2, axis=2)
C = -(D2 / r_0**2)**(beta_val / 2.0) / 2.0
f = np.dot(pinv(C), tec_screen[:, k, :].reshape(N_piercepoints))
for i, x in enumerate(xr[0: Nx]):
for j, y in enumerate(yr[0: Ny]):
p = calc_piercepoint(np.dot(np.array([x, y]), np.array([east, north])), up, height)
d2 = np.sum(np.square(pp[k, :, :] - p[0]), axis=1)
c = -(d2 / ( r_0**2 ))**(beta_val / 2.0) / 2.0
screen[i, j, k] = np.dot(c, f)
# Fit and remove a gradient.
# Plot gradient in lower-left corner with its own color bar?
if remove_gradient:
xs, ys = np.indices(screen.shape[0:2])
zs = screen[:, :, k]
XYZ = []
for xf, yf in zip(xs.flatten().tolist(), ys.flatten().tolist()):
XYZ.append([xf, yf, zs[xf, yf]])
XYZ = np.array(XYZ)
a, b, c = fitPLaneLTSQ(XYZ)
grad_plane = a * xs + b * ys + c
gradient[:, :, k] = grad_plane
screen[:, :, k] = screen[:, :, k] - grad_plane
screen[:, :, k] = screen[:, :, k] - np.mean(screen[:, :, k])
for t in range(fitted_tec1.shape[0]):
xs_pt = np.where(np.array(xr) > pp1[t, 0])[0][0]
ys_pt = np.where(np.array(yr) > pp1[t, 1])[0][0]
grad_plane_pt = a * xs_pt + b * ys_pt + c
fitted_tec1[t, k] = fitted_tec1[t, k] - grad_plane_pt
fitted_tec1[:, k] = fitted_tec1[:, k] - np.mean(fitted_tec1[:, k])
pbar.update(ipbar)
ipbar += 1
pbar.finish()
if min_tec is None:
vmin = np.min([np.amin(screen), np.amin(fitted_tec1)])
else:
vmin = min_tec
if max_tec is None:
vmax = np.max([np.amax(screen), np.amax(fitted_tec1)])
else:
vmax = max_tec
logging.info('Plotting TEC screens...')
fig, ax = plt.subplots(figsize=[7, 7])
pbar = progressbar.ProgressBar(maxval=N_times).start()
ipbar = 0
for k in range(N_times):
plt.clf()
im = plt.imshow(screen.transpose([1, 0, 2])[:, :, k],
cmap = plt.cm.jet,
origin = 'lower',
interpolation = 'nearest',
extent = (xr[0]/1000.0, xr[-1]/1000.0, yr[0]/1000.0, yr[-1]/1000.0),
vmin=vmin, vmax=vmax)
sm = plt.cm.ScalarMappable(cmap=plt.cm.jet,
norm=normalize(vmin=vmin, vmax=vmax))
sm._A = []
cbar = plt.colorbar(im)
cbar.set_label('TECU', rotation=270)
x = []
y = []
s = []
c = []
for j in range(fitted_tec1.shape[0]):
x.append(pp1[j, 0] / 1000.0)
y.append(pp1[j, 1] / 1000.0)
xs = np.where(np.array(xr) > pp1[j, 0])[0][0]
ys = np.where(np.array(yr) > pp1[j, 1])[0][0]
fit_screen_diff = abs(fitted_tec1[j, k] - screen[xs, ys, k])
s.append(max(20*fit_screen_diff/0.01, 10))
c.append(sm.to_rgba(fitted_tec1[j, k]))
plt.scatter(x, y, s=s, c=c)
if show_source_names:
labels = source_names
for label, xl, yl in zip(labels, x[0::N_stations], y[0::N_stations]):
plt.annotate(
label,
xy = (xl, yl), xytext = (-2, 2),
textcoords = 'offset points', ha = 'right', va = 'bottom')
plt.title('Screen {0}'.format(k))
plt.xlim(xr[-1]/1000.0, xr[0]/1000.0)
plt.ylim(yr[0]/1000.0, yr[-1]/1000.0)
plt.xlabel('Projected Distance along RA (km)')
plt.ylabel('Projected Distance along Dec (km)')
if remove_gradient:
axins = inset_axes(ax, width="15%", height="10%", loc=2)
axins.imshow(gradient.transpose([1, 0, 2])[:, : ,k],
cmap = plt.cm.jet,
origin = 'lower',
interpolation = 'nearest',
extent = (xr[0]/1000.0, xr[-1]/1000.0, yr[0]/1000.0, yr[-1]/1000.0),
vmin=vmin, vmax=vmax)
plt.xticks(visible=False)
plt.yticks(visible=False)
axins.set_xlim(xr[-1]/1000.0, xr[0]/1000.0)
axins.set_ylim(yr[0]/1000.0, yr[-1]/1000.0)
plt.savefig(root_dir+'/'+prestr+'frame%0.3i.png' % k)
pbar.update(ipbar)
ipbar += 1
pbar.finish()
plt.close(fig)
def _fit_phase_screen(station_names, source_names, pp, airmass, rr, weights,
times, height, order, r_0, beta, outQueue):
"""
Fits a screen to given phase values using Karhunen-Lo`eve base vectors
Parameters
----------
station_names: array
Array of station names
source_names: array
Array of source names
pp: array
Array of piercepoint locations
airmass: array
Array of airmass values (note: not currently used)
rr: array
Array of phase values to fit screen to
weights: array
Array of weights
times: array
Array of times
height: float
Height of screen (m)
order: int
Order of screen (i.e., number of KL base vectors to keep)
r_0: float
Scale size of phase fluctuations (m)
beta: float
Power-law index for phase structure function (5/3 => pure Kolmogorov
turbulence)
"""
import numpy as np
from pylab import kron, concatenate, pinv, norm, newaxis, find, amin, svd, eye
logging.info('Fitting screens...')
# Initialize arrays
N_stations = len(station_names)
N_sources = len(source_names)
N_times = len(times)
N_piercepoints = N_sources * N_stations
real_fit_white_all = np.zeros((N_times, N_sources, N_stations))
imag_fit_white_all = np.zeros((N_times, N_sources, N_stations))
phase_fit_white_all = np.zeros((N_times, N_sources, N_stations))
real_residual_all = np.zeros((N_times, N_sources, N_stations))
imag_residual_all = np.zeros((N_times, N_sources, N_stations))
phase_residual_all = np.zeros((N_times, N_sources, N_stations))
# Change phase to real/imag
rr_real = np.cos(rr)
rr_imag = np.sin(rr)
for k in range(N_times):
try:
D = np.resize(pp[k, :, :], (N_piercepoints, N_piercepoints, 3))
D = np.transpose(D, (1, 0, 2)) - D
D2 = np.sum(D**2, axis=2)
C = -(D2 / r_0**2)**(beta / 2.0) / 2.0
pinvC = pinv(C, rcond=1e-3)
U, S, V = svd(C)
invU = pinv(np.dot(np.transpose(U[:, :order]),
np.dot(weights[:, :, k], U[:, :order])),
rcond=1e-3)
# Calculate real screen
rr1 = np.dot(np.transpose(U[:, :order]),
np.dot(weights[:, :, k], rr_real[:, k]))
real_fit = np.dot(pinvC, np.dot(U[:, :order], np.dot(invU, rr1)))
real_fit_white_all[k, :, :] = real_fit.reshape(
(N_sources, N_stations))
residual = rr_real - np.dot(C, real_fit)[:, newaxis]
real_residual_all[k, :, :] = residual.reshape(
(N_sources, N_stations))
# Calculate imag screen
rr1 = np.dot(np.transpose(U[:, :order]),
np.dot(weights[:, :, k], rr_imag[:, k]))
imag_fit = np.dot(pinvC, np.dot(U[:, :order], np.dot(invU, rr1)))
imag_fit_white_all[k, :, :] = imag_fit.reshape(
(N_sources, N_stations))
residual = rr_imag - np.dot(C, imag_fit)[:, newaxis]
imag_residual_all[k, :, :] = residual.reshape(
(N_sources, N_stations))
# Calculate phase screen
phase_fit = np.dot(
pinvC, np.arctan2(np.dot(C, imag_fit), np.dot(C, real_fit)))
phase_fit_white_all[k, :, :] = phase_fit.reshape(
(N_sources, N_stations))
residual = rr - np.dot(C, phase_fit)[:, newaxis]
phase_residual_all[k, :, :] = residual.reshape(
(N_sources, N_stations))
except:
# Set screen to zero if fit did not work
logging.debug('Screen fit failed for timeslot {}'.format(k))
real_fit_white_all[k, :, :] = np.zeros((N_sources, N_stations))
real_residual_all[k, :, :] = np.ones((N_sources, N_stations))
imag_fit_white_all[k, :, :] = np.zeros((N_sources, N_stations))
imag_residual_all[k, :, :] = np.ones((N_sources, N_stations))
phase_fit_white_all[k, :, :] = np.zeros((N_sources, N_stations))
phase_residual_all[k, :, :] = np.ones((N_sources, N_stations))
outQueue.put([
real_fit_white_all, real_residual_all, imag_fit_white_all,
imag_residual_all, phase_fit_white_all, phase_residual_all, times
])
def makeTECparmdb(H, solset, TECsolTab, timewidths, freq, freqwidth):
"""Returns TEC screen parmdb parameters
H - H5parm object
solset - solution set with TEC screen parameters
TECsolTab = solution table with tecscreen values
timewidths - time widths of output parmdb
freq - frequency of output parmdb
freqwidth - frequency width of output parmdb
"""
from pylab import pinv
global ipbar, pbar
station_dict = H.getAnt(solset)
station_names = station_dict.keys()
station_positions = station_dict.values()
source_dict = H.getSou(solset)
source_names = source_dict.keys()
source_positions = source_dict.values()
tec_sf = solFetcher(TECsolTab)
tec_screen, axis_vals = tec_sf.getValues()
times = axis_vals['time']
beta = TECsolTab._v_attrs['beta']
r_0 = TECsolTab._v_attrs['r_0']
height = TECsolTab._v_attrs['height']
order = TECsolTab._v_attrs['order']
pp = tec_sf.t.piercepoint
N_sources = len(source_names)
N_times = len(times)
N_freqs = 1
N_stations = len(station_names)
N_piercepoints = N_sources * N_stations
freqs = freq
freqwidths = freqwidth
parms = {}
v = {}
v['times'] = times
v['timewidths'] = timewidths
v['freqs'] = freqs
v['freqwidths'] = freqwidths
for station_name in station_names:
for source_name in source_names:
v['values'] = np.zeros((N_times, N_freqs), dtype=np.double)
parmname = 'Piercepoint:X:%s:%s' % (station_name, source_name)
parms[parmname] = v.copy()
v['values'] = np.zeros((N_times, N_freqs), dtype=np.double)
parmname = 'Piercepoint:Y:%s:%s' % (station_name, source_name)
parms[parmname] = v.copy()
v['values'] = np.zeros((N_times, N_freqs), dtype=np.double)
parmname = 'Piercepoint:Z:%s:%s' % (station_name, source_name)
parms[parmname] = v.copy()
v['values'] = np.zeros((N_times, N_freqs), dtype=np.double)
parmname = 'TECfit_white:%s:%s' % (station_name, source_name)
parms[parmname] = v.copy()
v['values'] = np.zeros((N_times, N_freqs), dtype=np.double)
parmname = 'TECfit_white:0:%s:%s' % (station_name, source_name)
parms[parmname] = v.copy()
v['values'] = np.zeros((N_times, N_freqs), dtype=np.double)
parmname = 'TECfit_white:1:%s:%s' % (station_name, source_name)
parms[parmname] = v.copy()
for k in range(N_times):
D = np.resize(pp[k, :, :], (N_piercepoints, N_piercepoints, 3))
D = np.transpose(D, ( 1, 0, 2 )) - D
D2 = np.sum(D**2, axis=2)
C = -(D2 / (r_0**2))**(beta / 2.0) / 2.0
tec_fit_white = np.dot(pinv(C),
tec_screen[:, k, :].reshape(N_piercepoints))
pp_idx = 0
for src, source_name in enumerate(source_names):
for sta, station_name in enumerate(station_names):
parmname = 'Piercepoint:X:%s:%s' % (station_name, source_name)
parms[parmname]['values'][k, 0] = pp[k, pp_idx, 0]
parmname = 'Piercepoint:Y:%s:%s' % (station_name, source_name)
parms[parmname]['values'][k, 0] = pp[k, pp_idx, 1]
parmname = 'Piercepoint:Z:%s:%s' % (station_name, source_name)
parms[parmname]['values'][k, 0] = pp[k, pp_idx, 2]
parmname = 'TECfit_white:%s:%s' % (station_name, source_name)
parms[parmname]['values'][k, 0] = tec_fit_white[pp_idx]
parmname = 'TECfit_white:0:%s:%s' % (station_name, source_name)
parms[parmname]['values'][k, 0] = tec_fit_white[pp_idx]
parmname = 'TECfit_white:1:%s:%s' % (station_name, source_name)
parms[parmname]['values'][k, 0] = tec_fit_white[pp_idx]
pp_idx += 1
pbar.update(ipbar)
ipbar += 1
time_start = times[0] - timewidths[0]/2
time_end = times[-1] + timewidths[-1]/2
v['times'] = np.array([(time_start + time_end) / 2])
v['timewidths'] = np.array([time_end - time_start])
v_r0 = v.copy()
v_r0['values'] = np.array(r_0, dtype=np.double, ndmin=2)
parms['r_0'] = v_r0
v_beta = v.copy()
v_beta['values'] = np.array(beta, dtype=np.double, ndmin=2)
parms['beta'] = v_beta
v_height = v.copy()
v_height['values'] = np.array(height, dtype=np.double, ndmin=2)
parms['height'] = v_height
return parms
def _fit_screen(station_names, source_names, full_matrices, pp, rr, weights, order, r_0, beta,
screen_type):
"""
Fits a screen to amplitudes or phases using Karhunen-Lo`eve base vectors
Parameters
----------
station_names: array
Array of station names
source_names: array
Array of source names
full_matrices : list of arrays
List of [C, pivC, U] matrices for all piercepoints
pp: array
Array of piercepoint locations
airmass: array
Array of airmass values (note: not currently used)
rr: array
Array of amp values to fit screen to
weights: array
Array of weights
order: int
Order of screen (i.e., number of KL base vectors to keep)
r_0: float
Scale size of amp fluctuations (m)
beta: float
Power-law index for amp structure function (5/3 => pure Kolmogorov
turbulence)
screen_type : str
Type of screen: 'phase' or 'amplitude'
Returns
-------
screen_fit_white_all, screen_residual_all : array, array
Arrays of screen and residual (actual - screen) values
"""
import numpy as np
from pylab import kron, concatenate, pinv, norm, newaxis, find, amin, svd, eye
# Identify flagged directions
N_sources_all = len(source_names)
unflagged = np.where(weights > 0.0)
N_sources = len(source_names[unflagged])
# Initialize arrays
N_stations = len(station_names)
N_piercepoints = N_sources * N_stations
N_piercepoints_all = N_sources_all * N_stations
screen_fit_all = np.zeros((N_sources_all, N_stations))
pp_all = pp.copy()
rr_all = rr.copy()
pp = pp_all[unflagged[0], :]
w = np.diag(weights[unflagged])
# Calculate matrices
if N_sources == N_sources_all:
C, pinvC, U = full_matrices
else:
# Recalculate for unflagged directions
C, pinvC, U = _calculate_svd(pp, r_0, beta, N_piercepoints)
invU = pinv(np.dot(np.transpose(U[:, :order]), np.dot(w, U)[:, :order]), rcond=1e-3)
# Fit screen to unflagged directions
if screen_type == 'phase':
# Change phase to real/imag
rr_real = np.cos(rr[unflagged])
rr_imag = np.sin(rr[unflagged])
# Calculate real screen
rr1 = np.dot(np.transpose(U[:, :order]), np.dot(w, rr_real))
real_fit = np.dot(pinvC, np.dot(U[:, :order], np.dot(invU, rr1)))
# Calculate imag screen
rr1 = np.dot(np.transpose(U[:, :order]), np.dot(w, rr_imag))
imag_fit = np.dot(pinvC, np.dot(U[:, :order], np.dot(invU, rr1)))
# Calculate phase screen
screen_fit = np.arctan2(np.dot(C, imag_fit), np.dot(C, real_fit))
screen_fit_white = np.dot(pinvC, screen_fit)
elif screen_type == 'amplitude':
# Calculate log(amp) screen
rr = rr[unflagged]
rr1 = np.dot(np.transpose(U[:, :order]), np.dot(w, np.log10(rr)))
amp_fit_log = np.dot(pinvC, np.dot(U[:, :order], np.dot(invU, rr1)))
# Calculate amp screen
screen_fit = 10**(np.dot(C, amp_fit_log))
screen_fit_white = np.dot(pinvC, screen_fit)
elif screen_type == 'tec':
# Calculate tec screen
rr = rr[unflagged]
rr1 = np.dot(np.transpose(U[:, :order]), np.dot(w, rr))
tec_fit = np.dot(pinvC, np.dot(U[:, :order], np.dot(invU, rr1)))
# Calculate amp screen
screen_fit = np.dot(C, tec_fit)
screen_fit_white = np.dot(pinvC, screen_fit)
# Calculate screen in all directions
if N_sources != N_sources_all:
screen_fit_all[unflagged[0], :] = screen_fit[:, newaxis]
flagged = np.where(weights <= 0.0)
for findx in flagged[0]:
p = pp_all[findx, :]
d2 = np.sum(np.square(pp - p), axis=1)
c = -(d2 / ( r_0**2 ))**(beta / 2.0) / 2.0
screen_fit_all[findx, :] = np.dot(c, screen_fit_white)
C, pinvC, U = full_matrices
screen_fit_white_all = np.dot(pinvC, screen_fit_all)
screen_residual_all = rr_all - screen_fit_all.reshape(N_piercepoints_all)
else:
screen_fit_white_all = screen_fit_white
screen_residual_all = rr_all - np.dot(C, screen_fit_white)
screen_fit_white_all = screen_fit_white_all.reshape((N_sources_all, N_stations))
screen_residual_all = screen_residual_all.reshape((N_sources_all, N_stations))
return (screen_fit_white_all, screen_residual_all)
continue
unknown_columns = sorted(
list(set(range(len(all_cids))).difference(known_columns))
) # find all the indices of columns not in known_columns
unknown_rows = find(isnan(dG0_r))
known_rows = sorted(list(set(range(len(reactions))).difference(
unknown_rows))) # find all the indices of rows with measured dG0_r
S_measured = S[known_rows, :]
b = dG0_r[known_rows] - dot(S_measured[:, known_columns], dG0_f[known_columns])
S_red = S_measured[:, unknown_columns]
print "Formation energies from from GC and from linear regression: "
# linear regression to solve the missing formation energies (S*x = b)
inv_corr_mat = pinv(dot(S_red.T, S_red))
x = dot(dot(inv_corr_mat, S_red.T), b)
for c in xrange(len(unknown_columns)):
dG0_f[unknown_columns[c], 0] = x[c, 0]
csv_out = csv.writer(open('../res/acetogens_compounds.csv', 'w'))
for c in xrange(len(all_cids)):
if (c in known_columns):
csv_out.writerow([
'C%05d' % all_cids[c],
'%8.1f' % dG0_f[c, 0], 'Group Contribution'
])
else:
csv_out.writerow(
['C%05d' % all_cids[c],
'%8.1f' % dG0_f[c, 0], 'Calculated'])
def add_stations(station_selection, phases0, phases1, flags, mask,
station_names, station_positions, source_names, source_selection,
times, freqs, r, nband_min=2, soln_type='phase', nstations_max=None,
excluded_stations=None, t_step=5, tec_step1=5, tec_step2=21,
search_full_tec_range=True):
"""
Adds stations to TEC fitting using an iterative initial-guess search to
ensure the global min is found
Keyword arguments:
station_selection -- indices of stations to use in fitting
phases0 -- XX phase solutions
phases1 -- YY phase solutions
flags -- phase solution flags (0 = use, 1 = flagged)
mask -- mask for sources and frequencies (0 = ignore, 1 = use)
station_names -- array of station names
source_names -- array of source names
source_selection -- indices of sources to use in fitting
times -- array of times
freqs -- array of frequencies
r -- array of TEC solutions returned by fit_tec_per_source_pair()
nband_min -- min number of bands for a source to be used
soln_type -- type of phase solution: 'phase' or 'scalarphase'
nstations_max -- max number of stations to use
excluded_stations -- stations to exclude
t_step -- try full TEC range every t_step number of solution times
tec_step1 -- number of steps in TEC subrange (+/- last TEC fit value)
tec_step2 -- number of steps in full TEC range (-0.1 -- 0.1)
search_full_tec_range -- always search the full TEC range (-0.1 -- 0.1)
"""
from pylab import pinv, newaxis, find, amin
import numpy as np
from lofar.expion import baselinefitting
try:
import progressbar
except ImportError:
import losoto.progressbar as progressbar
N_sources_selected = len(source_selection)
N_stations_selected = len(station_selection)
N_piercepoints = N_sources_selected * N_stations_selected
N_times = len(times)
N_stations = len(station_names)
N_sources = len(source_names)
N_pairs = 0
for ii, i in enumerate(source_selection):
for jj, j in enumerate(source_selection):
if j == i:
break
subband_selection = find(mask[i,:] * mask[j,:])
if len(subband_selection) < nband_min:
continue
N_pairs += 1
D = np.resize(station_positions, (N_stations, N_stations, 3))
D = np.transpose(D, (1, 0, 2)) - D
D = np.sqrt(np.sum(D**2, axis=2))
station_selection1 = station_selection
stations_to_add = np.array([i for i in xrange(len(station_names))
if i not in station_selection1 and station_names[i] not in
excluded_stations])
if len(stations_to_add) == 0:
return station_selection1, r
# Check if desired number of stations is already reached
if nstations_max is not None:
if len(station_selection1) >= nstations_max:
return station_selection1, r
logging.info("Using fitting with iterative search for remaining stations "
"(up to {0} stations in total)".format(nstations_max))
q = r
while len(stations_to_add)>0:
D1 = D[stations_to_add[:,newaxis], station_selection1[newaxis,:]]
minimum_distance = amin(D1, axis=1)
station_to_add = stations_to_add[np.argmin(minimum_distance)]
station_selection1 = np.append(station_selection1, station_to_add)
N_stations_selected1 = len(station_selection1)
# Remove station from list
stations_to_add = stations_to_add[stations_to_add != station_to_add]
sols_list = []
eq_list = []
min_e_list = []
ipbar = 0
logging.info('Fitting TEC values with {0} included...'.format(
station_names[station_to_add]))
pbar = progressbar.ProgressBar(maxval=N_pairs*N_times).start()
for ii, i in enumerate(source_selection):
for jj, j in enumerate(source_selection):
if j == i:
break
subband_selection = find(mask[i,:] * mask[j,:])
if len(subband_selection) < nband_min:
continue
logging.debug('Adding {0} for source pair: {1}-{2}'.format(
station_names[station_to_add], i, j))
p0 = phases0[i, station_selection1[:,newaxis],
subband_selection[newaxis,:], :] - phases0[j,
station_selection1[:,newaxis], subband_selection[newaxis,:], :]
p0 = p0 - np.mean(p0, axis=0)[newaxis,:,:]
if soln_type != 'scalarphase':
p1 = phases1[i, station_selection1[:,newaxis],
subband_selection[newaxis,:], :] - phases1[j,
station_selection1[:,newaxis], subband_selection[newaxis,:], :]
p1 = p1 - np.mean(p1, axis=0)[newaxis, :, :]
A = np.zeros((len(subband_selection), 1))
A[:, 0] = 8.44797245e9 / freqs[subband_selection]
sd = (0.1 * 30e6) / freqs[subband_selection] # standard deviation of phase solutions as function of frequency (rad)
flags_source_pair = flags[i, station_selection1[:,newaxis],
subband_selection[newaxis,:], :] * flags[j,
station_selection1[:,newaxis], subband_selection[newaxis,:], :]
constant_parms = np.zeros((1, N_stations_selected1), dtype = np.bool)
sols = np.zeros((N_times, N_stations_selected1), dtype = np.float)
p_0_best = None
for t_idx in xrange(N_times):
if np.mod(t_idx, t_step) == 0:
min_e = np.Inf
if p_0_best is not None and not search_full_tec_range:
min_tec = p_0_best[0, -1] - 0.02
max_tec = p_0_best[0, -1] + 0.02
nsteps = tec_step1
else:
min_tec = -0.1
max_tec = 0.1
nsteps = tec_step2
logging.debug(' Trying initial guesses between {0} and '
'{1} TECU'.format(min_tec, max_tec))
for offset in np.linspace(min_tec, max_tec, nsteps):
p_0 = np.zeros((1, N_stations_selected1), np.double)
p_0[0, :N_stations_selected1-1] = (q[ii, t_idx, :] -
q[jj, t_idx, :])[newaxis,:]
p_0[0, -1] = offset
x = p0[:,:,t_idx].copy()
f = flags_source_pair[:, :, t_idx].copy()
sol0 = baselinefitting.fit(x.T, A, p_0, f, constant_parms)
sol0 -= np.mean(sol0)
residual = np.mod(np.dot(A, sol0) - x.T + np.pi,
2 * np.pi) - np.pi
e = np.var(residual[f.T==0])
if soln_type != 'scalarphase':
x = p1[:,:,t_idx].copy()
f = flags_source_pair[:, :, t_idx].copy()
sol1 = baselinefitting.fit(x.T, A, p_0, f,
constant_parms)
sol1 -= np.mean(sol1)
residual = np.mod(np.dot(A, sol1) - x.T + np.pi,
2 * np.pi) - np.pi
residual = residual[f.T==0]
e += np.var(residual)
else:
sol1 = sol0
if e < min_e:
logging.debug(' Found new min variance of {0} '
'with initial guess of {1} TECU'.format(e,
p_0[0, -1]))
min_e = e
p_0_best = p_0
sols[t_idx, :] = (sol0[0, :] + sol1[0, :])/2
else:
# Use previous init
x = p0[:, :, t_idx].copy()
f = flags_source_pair[:, :, t_idx].copy()
sol0 = baselinefitting.fit(x.T, A, p_0_best, f,
constant_parms)
sol0 -= np.mean(sol0)
if soln_type != 'scalarphase':
x = p1[:, :, t_idx].copy()
f = flags_source_pair[:, :, t_idx].copy()
sol1 = baselinefitting.fit(x.T, A, p_0_best, f,
constant_parms)
sol1 -= np.mean(sol1)
else:
sol1 = sol0
sols[t_idx, :] = (sol0[0, :] + sol1[0, :])/2
ipbar += 1
pbar.update(ipbar)
### Remove outliers
logging.debug(' Searching for outliers...')
for kk in xrange(10):
s = sols[:, -1].copy()
selection = np.zeros(len(s), np.bool)
for t_idx in xrange(len(s)):
start_idx = np.max([t_idx-10, 0])
end_idx = np.min([t_idx+10, len(s)])
selection[t_idx] = np.sum(abs(s[start_idx:end_idx] -
s[t_idx]) < 0.02) > (end_idx - start_idx - 8)
outliers = find(np.logical_not(selection))
if len(outliers) == 0:
break
for t_idx in outliers:
try:
idx0 = find(selection[:t_idx])[-1]
except IndexError:
idx0 = -1
try:
idx1 = find(selection[t_idx+1:])[0] + t_idx + 1
except IndexError:
idx1 = -1
if idx0 == -1:
s[t_idx] = s[idx1]
elif idx1 == -1:
s[t_idx] = s[idx0]
else:
s[t_idx] = (s[idx0] * (idx1-t_idx) + s[idx1] *
(t_idx-idx0)) / (idx1-idx0)
p_0 = np.zeros((1, N_stations_selected1), np.double)
p_0[0,:] = sols[t_idx,:]
p_0[0,-1] = s[t_idx]
x = p0[:,:,t_idx].copy()
f = flags_source_pair[:,:,t_idx].copy()
sol0 = baselinefitting.fit(x.T, A, p_0, f, constant_parms)
sol0 -= np.mean(sol0)
if soln_type != 'scalarphase':
x = p1[:,:,t_idx].copy()
sol1 = baselinefitting.fit(x.T, A, p_0, f,
constant_parms)
sol1 -= np.mean(sol1)
else:
sol1 = sol0
sols[t_idx, :] = (sol0[0,:] + sol1[0,:])/2
weight = 1.0
sols_list.append(weight*sols)
min_e_list.append(min_e)
eq = np.zeros(N_sources)
eq[ii] = weight
eq[jj] = -weight
eq_list.append(eq)
sols = np.array(sols_list)
B = np.array(eq_list)
pinvB = pinv(B)
q = np.dot(pinvB, sols.transpose([1,0,2]))
pbar.finish()
if nstations_max is not None:
if N_stations_selected1 == nstations_max:
break
return station_selection1, q
def fit_screen_to_tec(station_names, source_names, pp, airmass, rr, times,
height, order, r_0, beta):
"""
Fits a screen to given TEC values using Karhunen-Lo`eve base vectors
Keyword arguments:
station_names -- array of station names
source_names -- array of source names
pp -- array of piercepoint locations
airmass -- array of airmass values
rr -- array of TEC solutions
times -- array of times
height -- height of screen (m)
order -- order of screen (i.e., number of KL base vectors to keep)
r_0 -- scale size of phase fluctuations (m)
beta -- power-law index for phase structure function (5/3 =>
pure Kolmogorov turbulence)
"""
import numpy as np
from pylab import kron, concatenate, pinv, norm, newaxis, find, amin, svd, eye
try:
import progressbar
except ImportError:
import losoto.progressbar as progressbar
logging.info('Fitting screens to TEC values...')
N_stations = len(station_names)
N_sources = len(source_names)
N_times = len(times)
tec_fit_all = np.zeros((N_times, N_sources, N_stations))
residual_all = np.zeros((N_times, N_sources, N_stations))
A = concatenate([
kron(eye(N_sources), np.ones((N_stations, 1))),
kron(np.ones((N_sources, 1)), eye(N_stations))
],
axis=1)
N_piercepoints = N_sources * N_stations
P = eye(N_piercepoints) - np.dot(np.dot(A, pinv(np.dot(A.T, A))), A.T)
pbar = progressbar.ProgressBar(maxval=N_times).start()
ipbar = 0
for k in range(N_times):
try:
D = np.resize(pp[k, :, :], (N_piercepoints, N_piercepoints, 3))
D = np.transpose(D, (1, 0, 2)) - D
D2 = np.sum(D**2, axis=2)
C = -(D2 / r_0**2)**(beta / 2.0) / 2.0
P1 = eye(N_piercepoints) - np.ones(
(N_piercepoints, N_piercepoints)) / N_piercepoints
C1 = np.dot(np.dot(P1, C), P1)
U, S, V = svd(C1)
B = np.dot(P, np.dot(np.diag(airmass[k, :]), U[:, :order]))
pinvB = pinv(B, rcond=1e-3)
rr1 = np.dot(P, rr[:, k])
tec_fit = np.dot(U[:, :order], np.dot(pinvB, rr1))
tec_fit_all[k, :, :] = tec_fit.reshape((N_sources, N_stations))
residual = rr1 - np.dot(P, tec_fit)
residual_all[k, :, :] = residual.reshape((N_sources, N_stations))
except:
# Set screen to zero if fit did not work
logging.debug('Tecscreen fit failed for timeslot {0}'.format(k))
tec_fit_all[k, :, :] = np.zeros((N_sources, N_stations))
residual_all[k, :, :] = np.ones((N_sources, N_stations))
pbar.update(ipbar)
ipbar += 1
pbar.finish()
return tec_fit_all, residual_all
def _fit_screen(station_names, source_names, full_matrices, pp, rr, weights,
order, r_0, beta, screen_type):
"""
Fits a screen to amplitudes or phases using Karhunen-Lo`eve base vectors
Parameters
----------
station_names: array
Array of station names
source_names: array
Array of source names
full_matrices : list of arrays
List of [C, pivC, U] matrices for all piercepoints
pp: array
Array of piercepoint locations
airmass: array
Array of airmass values (note: not currently used)
rr: array
Array of amp values to fit screen to
weights: array
Array of weights
order: int
Order of screen (i.e., number of KL base vectors to keep)
r_0: float
Scale size of amp fluctuations (m)
beta: float
Power-law index for amp structure function (5/3 => pure Kolmogorov
turbulence)
screen_type : str
Type of screen: 'phase' or 'amplitude'
Returns
-------
screen_fit_white_all, screen_residual_all : array, array
Arrays of screen and residual (actual - screen) values
"""
import numpy as np
from pylab import kron, concatenate, pinv, norm, newaxis, find, amin, svd, eye
# Identify flagged directions
N_sources_all = len(source_names)
unflagged = np.where(weights > 0.0)
N_sources = len(source_names[unflagged])
# Initialize arrays
N_stations = len(station_names)
N_piercepoints = N_sources * N_stations
N_piercepoints_all = N_sources_all * N_stations
screen_fit_all = np.zeros((N_sources_all, N_stations))
pp_all = pp.copy()
rr_all = rr.copy()
pp = pp_all[unflagged[0], :]
w = np.diag(weights[unflagged])
# Calculate matrices
if N_sources == N_sources_all:
C, pinvC, U = full_matrices
else:
# Recalculate for unflagged directions
C, pinvC, U = _calculate_svd(pp, r_0, beta, N_piercepoints)
invU = pinv(np.dot(np.transpose(U[:, :order]),
np.dot(w, U)[:, :order]),
rcond=1e-3)
# Fit screen to unflagged directions
if screen_type == 'phase':
# Change phase to real/imag
rr_real = np.cos(rr[unflagged])
rr_imag = np.sin(rr[unflagged])
# Calculate real screen
rr1 = np.dot(np.transpose(U[:, :order]), np.dot(w, rr_real))
real_fit = np.dot(pinvC, np.dot(U[:, :order], np.dot(invU, rr1)))
# Calculate imag screen
rr1 = np.dot(np.transpose(U[:, :order]), np.dot(w, rr_imag))
imag_fit = np.dot(pinvC, np.dot(U[:, :order], np.dot(invU, rr1)))
# Calculate phase screen
screen_fit = np.arctan2(np.dot(C, imag_fit), np.dot(C, real_fit))
screen_fit_white = np.dot(pinvC, screen_fit)
else:
# Calculate log(amp) screen
rr = rr[unflagged]
rr1 = np.dot(np.transpose(U[:, :order]), np.dot(w, np.log10(rr)))
amp_fit_log = np.dot(pinvC, np.dot(U[:, :order], np.dot(invU, rr1)))
# Calculate amp screen
screen_fit = 10**(np.dot(C, amp_fit_log))
screen_fit_white = np.dot(pinvC, screen_fit)
# Calculate screen in all directions
if N_sources != N_sources_all:
screen_fit_all[unflagged[0], :] = screen_fit[:, newaxis]
flagged = np.where(weights <= 0.0)
for findx in flagged[0]:
p = pp_all[findx, :]
d2 = np.sum(np.square(pp - p), axis=1)
c = -(d2 / (r_0**2))**(beta / 2.0) / 2.0
screen_fit_all[findx, :] = np.dot(c, screen_fit_white)
C, pinvC, U = full_matrices
screen_fit_white_all = np.dot(pinvC, screen_fit_all)
screen_residual_all = rr_all - screen_fit_all.reshape(
N_piercepoints_all)
else:
screen_fit_white_all = screen_fit_white
screen_residual_all = rr_all - np.dot(C, screen_fit_white)
screen_fit_white_all = screen_fit_white_all.reshape(
(N_sources_all, N_stations))
screen_residual_all = screen_residual_all.reshape(
(N_sources_all, N_stations))
return (screen_fit_white_all, screen_residual_all)
def calcSlipParams3D2(IC, m, FS, ymin, T, P0 = [14000., 1.16, 1., 0., 0.]):
"""
calculates a set of SLIP parameters that result in the desired motion.
:args:
IC (3x float) : initial condition y, vx, vz
FS (3x float) : final state y, vx, vz
ymin : minimal vertical excursion
T : total step duration
P0 (5x float) : initial guess for parameters [k, alpha, L0, beta, dE]
(dE is ignored. It is calculated from IC and FS)
:returns:
[k, alpha, L0, beta, dE] parameter vector that results in the desired state
"""
dE = (FS[0]-IC[0])*m*9.81 + .5*m*(FS[1]**2 + FS[2]**2
- IC[1]**2 - IC[2]**2)
k, alpha, L0, beta, _ = P0
def getDiff(t, y):
""" returns the difference in desired params """
delta = [T - t[-1],
ymin - min(y[:,1]),
#FS - y[-1,[1,3,5]],
FS[[0,2]] - y[-1,[1,5]]
]
return hstack(delta)
rcond = 1e-3 # for pinverting the jacobian. this will be adapted during the process
init_success = False
while not init_success:
try:
pars = [k, L0, m, alpha, beta, dE]
t, y = sl.qSLIP_step3D(IC, pars)
init_success = True
except ValueError:
L0 -= .02
d0 = getDiff(t, y)
nd0 = norm(d0)
#print \"difference: \", nd0, d0
cancel = False
niter = 0
while nd0 > 1e-6 and not cancel:
niter += 1
if niter > 20:
print "too many iterations"
cancel = True
break
# calculate jacobian dDelta / dP
#print \"rep:\", niter
hs = array([10, .0001, .0001, .0001])
pdims = [0, 1, 3, 4]
J = []
for dim in range(4):
parsp = [k, L0, m, alpha, beta, dE][:]
parsm = [k, L0, m, alpha, beta, dE][:]
parsp[pdims[dim]] += hs[dim]
parsm[pdims[dim]] -= hs[dim]
t, y = sl.qSLIP_step3D(IC, parsm)
dm = getDiff(t, y)
# circumvent "too low starting condition":
try:
t, y = sl.qSLIP_step3D(IC, parsp)
dp = getDiff(t, y)
J.append((dp - dm)/(2*hs[dim]))
except ValueError:
# run unilateral derivative instead
parsp = [k, L0, m, alpha, beta, dE]
t, y = sl.qSLIP_step3D(IC, parsm)
dp = getDiff(t, y)
J.append((dp - dm)/(hs[dim]))
print "|1>"
J = vstack(J).T
pred = [k, L0, alpha, beta]
update = dot(pinv(J, rcond=rcond), d0)
nrm = norm(update * array([.001, 10, 10, 10]))
success = False
cancel = False
rep = 0
while not (success or cancel):
rep += 1
#print \"update:\", update
if rep > 3:
# reset
if rcond < 1e-10:
#print \"rcond too small!\"
cancel = True
else:
rcond /= 100
pars = [k, L0, m, alpha, beta, dE]
break
dk, dL0, dalpha, dbeta = update
pars = [k - dk, L0 - dL0, m, alpha - dalpha, beta - dbeta, dE]
try:
t, y = sl.qSLIP_step3D(IC, pars)
d0 = getDiff(t, y)
#print "d0 = ", norm(d0)
except (ValueError, sl.SimFailError, TypeError):
#print "escaping ..."
update /= 2
continue
if norm(d0) < nd0:
success = True
nd0 = norm(d0)
else:
update /= 2
#print \"difference: \", norm(d0), d0
k, L0, alpha, beta = array(pars)[[0,1,3,4]]
#if not cancel:
# print \"converged!\"
return array([k, alpha, L0, beta, dE])
