def getTECBaselineFit(ph,
amp,
freqs,
SBselect,
polIdx,
stIdx,
useOffset=False,
stations=[],
initSol=[],
chi2cut=300.,
timeIdx=0):
global tecarray
global offsetarray
global residualarray
amp[np.isnan(ph)] = 1
ph[np.isnan(ph)] = 0.
ph = np.unwrap(ph, axis=0)
#first unwrap data and get initial guess
nT = ph.shape[0]
nF = freqs.shape[0]
nSt = ph.shape[2]
nparms = 1 + (useOffset > 0)
sol = np.zeros((nSt, nparms), dtype=np.float)
A = np.zeros((nF, nparms), dtype=np.float)
A[:, 0] = -8.44797245e9 / freqs
if useOffset:
A[:, 1] = np.ones((nF, ))
# init first sol
big_array = (np.arange(-0.1, 0.1, 0.005) * A[:, 0][:, np.newaxis]).T
diff = np.sum(np.absolute(
np.remainder(
big_array[:, :, np.newaxis] -
(ph[0, :, :] - ph[0, :, :][:, [0]]) + np.pi, 2 * np.pi) - np.pi),
axis=1)
init_idx = np.argmin(diff, axis=0)
sol[:, 0] = init_idx * 0.005 - 0.1
print("Initializing with", sol[:, 0])
for itm in range(nT):
if itm % 100 == 0 and itm > 0:
sys.stdout.write(
str(itm) + '... ' + str(sol[-1, 0] - sol[0, 0]) + ' ' +
str(sol[-1, -1] - sol[0, -1]) + ' ')
sys.stdout.flush()
flags = (amp[itm, :] == 1)
nrFlags = np.sum(flags, axis=0)
sol = fitting.fit(ph[itm], A, sol.T, flags).T
tecarray[itm + timeIdx, stIdx, polIdx] = sol[:, 0]
if useOffset:
offsetarray[itm + timeIdx, stIdx, polIdx] = sol[:, 1]
residual = ph[itm] - np.dot(A, sol.T)
residual = residual - residual[:, 0][:, np.newaxis]
residual = np.remainder(residual + np.pi, 2 * np.pi) - np.pi
residual[flags] = 0
residualarray[np.ix_([itm + timeIdx], SBselect, stIdx,
[polIdx])] = residual.reshape((1, nF, nSt, 1))
def getResidualPhaseWraps(avgResiduals, freqs):
flags = np.average(avgResiduals.mask, axis=1) > 0.5
nSt = avgResiduals.shape[1]
nF = freqs.shape[0]
wraps = np.zeros((nSt, ), dtype=np.float)
tmpflags = flags
tmpfreqs = freqs[np.logical_not(tmpflags)]
(tmpbasef, steps) = getPhaseWrapBase(tmpfreqs)
basef = np.zeros(freqs.shape)
basef[np.logical_not(tmpflags)] = tmpbasef
basef = basef.reshape((-1, 1))
data = avgResiduals[:, :]
wraps = fitting.fit(data, basef, wraps, flags).flatten()
return (wraps, steps)
def getResidualPhaseWraps(avgResiduals, freqs):
flags = np.average(avgResiduals.mask, axis=1) > 0.5
nSt = avgResiduals.shape[1]
nF = freqs.shape[0]
wraps = np.zeros((nSt,), dtype=np.float)
tmpflags = flags
tmpfreqs = freqs[np.logical_not(tmpflags)]
(tmpbasef, steps) = getPhaseWrapBase(tmpfreqs)
basef = np.zeros(freqs.shape)
basef[np.logical_not(tmpflags)] = tmpbasef
basef = basef.reshape((-1, 1))
data = avgResiduals[:, :]
wraps = fitting.fit(data, basef, wraps, flags).flatten()
return (wraps, steps)
def getResidualPhaseWraps2(avgResiduals, freqs):
flags = avgResiduals[:, 10] == 0.
nSt = avgResiduals.shape[1]
nF = freqs.shape[0]
wraps = np.zeros((nSt, ), dtype=np.float)
#tmpflags=np.sum(flags[:,np.sum(flags,axis=0)<(nF*0.5)],axis=1)
tmpflags = flags
tmpfreqs = freqs[np.logical_not(tmpflags)]
tmpbasef, steps = getPhaseWrapBase(tmpfreqs)
basef = np.zeros(freqs.shape)
basef[np.logical_not(tmpflags)] = tmpbasef
basef = basef.reshape((-1, 1))
data = avgResiduals[:, :]
wraps = fitting.fit(data, basef, wraps, flags).flatten()
return wraps, steps
def getResidualPhaseWraps(avgResiduals,freqs):
flags=avgResiduals[:,10]==0.
nSt=avgResiduals.shape[1]
nF=freqs.shape[0]
wraps=np.zeros((nSt,),dtype=np.float)
#tmpflags=np.sum(flags[:,np.sum(flags,axis=0)<(nF*0.5)],axis=1)
tmpflags=flags
tmpfreqs=freqs[np.logical_not(tmpflags)]
tmpbasef,steps=getPhaseWrapBase(tmpfreqs)
basef=np.zeros(freqs.shape)
basef[np.logical_not(tmpflags)]=tmpbasef
basef=basef.reshape((-1,1))
data=avgResiduals[:,:]
wraps=fitting.fit(data,basef,wraps,flags).flatten()
return wraps,steps
def getResidualPhaseWraps(avgResiduals, freqs):
flags = np.average(avgResiduals.mask,axis=1)>0.5
nSt = avgResiduals.shape[1]
nF = freqs.shape[0]
wraps = np.zeros((nSt, ), dtype=np.float)
tmpflags = flags
tmpfreqs = freqs[np.logical_not(tmpflags)]
steps=[0,0]
if np.ma.count(tmpfreqs) < 3:
logging.debug('Cannot unwrap, too many channels flagged')
return (wraps,steps)
(tmpbasef, steps) = getPhaseWrapBase(tmpfreqs)
basef = np.zeros(freqs.shape)
basef[np.logical_not(tmpflags)] = tmpbasef
basef = basef.reshape((-1, 1))
data = avgResiduals[:, :]
if has_fitting:
wraps = fitting.fit(data, basef, wraps, flags).flatten()
return (wraps, steps)
def getResidualPhaseWraps(avgResiduals, freqs):
flags = np.average(avgResiduals.mask,axis=1)>0.5
nSt = avgResiduals.shape[1]
nF = freqs.shape[0]
wraps = np.zeros((nSt, ), dtype=np.float)
tmpflags = flags
tmpfreqs = freqs[np.logical_not(tmpflags)]
steps=[0,0]
if np.ma.count(tmpfreqs) < 3:
logging.debug('Cannot unwrap, too many channels flagged')
return (wraps,steps)
(tmpbasef, steps) = getPhaseWrapBase(tmpfreqs)
basef = np.zeros(freqs.shape)
basef[np.logical_not(tmpflags)] = tmpbasef
basef = basef.reshape((-1, 1))
data = avgResiduals[:, :]
if has_fitting:
wraps = fitting.fit(data, basef, wraps, flags).flatten()
return (wraps, steps)
def getTECBaselineFit(ph,amp,freqs,SBselect,polIdx,stIdx,useOffset=False,stations=[],initSol=[],chi2cut=300.,timeIdx=0):
global tecarray
global offsetarray
global residualarray
amp[np.isnan(ph)]=1
ph[np.isnan(ph)]=0.
ph=np.unwrap(ph,axis=0)
#first unwrap data and get initial guess
nT=ph.shape[0]
nF=freqs.shape[0]
nSt=ph.shape[2]
nparms=1+(useOffset>0)
sol = np.zeros((nSt,nparms),dtype=np.float)
A=np.zeros((nF,nparms),dtype=np.float)
A[:,0] = -8.44797245e9/freqs
if useOffset:
A[:,1] = np.ones((nF,))
# init first sol
big_array=(np.arange(-0.1,0.1,0.005)*A[:,0][:,np.newaxis]).T
diff=np.sum(np.absolute(np.remainder(big_array[:,:,np.newaxis]-(ph[0,:,:]-ph[0,:,:][:,[0]])+np.pi,2*np.pi)-np.pi),axis=1)
init_idx=np.argmin(diff,axis=0)
sol[:,0]=init_idx*0.005-0.1
print "Initializing with",sol[:,0]
for itm in range(nT):
if itm%100==0 and itm>0:
sys.stdout.write(str(itm)+'... '+str(sol[-1,0]-sol[0,0])+' '+str(sol[-1,-1]-sol[0,-1])+' ')
sys.stdout.flush()
flags=(amp[itm,:]==1);
nrFlags=np.sum(flags,axis=0)
sol=fitting.fit(ph[itm],A,sol.T,flags).T
tecarray[itm+timeIdx,stIdx,polIdx]=sol[:,0]
if useOffset:
offsetarray[itm+timeIdx,stIdx,polIdx]=sol[:,1]
residual = ph[itm] - np.dot(A, sol.T)
residual = residual - residual[:, 0][:,np.newaxis]
residual = np.remainder(residual+np.pi, 2*np.pi) - np.pi
residual[flags]=0
residualarray[np.ix_([itm+timeIdx],SBselect,stIdx,[polIdx])]=residual.reshape((1,nF,nSt,1))
def add_stations(station_selection, phases0, phases1, flags, mask,
station_names, station_positions, source_names, source_selection,
times, freqs, r, nband_min=2, soln_type='phase', nstations_max=None,
excluded_stations=None, t_step=5, tec_step1=5, tec_step2=21,
search_full_tec_range=True):
"""
Adds stations to TEC fitting using an iterative initial-guess search to
ensure the global min is found
Keyword arguments:
station_selection -- indices of stations to use in fitting
phases0 -- XX phase solutions
phases1 -- YY phase solutions
flags -- phase solution flags (0 = use, 1 = flagged)
mask -- mask for sources and frequencies (0 = ignore, 1 = use)
station_names -- array of station names
source_names -- array of source names
source_selection -- indices of sources to use in fitting
times -- array of times
freqs -- array of frequencies
r -- array of TEC solutions returned by fit_tec_per_source_pair()
nband_min -- min number of bands for a source to be used
soln_type -- type of phase solution: 'phase' or 'scalarphase'
nstations_max -- max number of stations to use
excluded_stations -- stations to exclude
t_step -- try full TEC range every t_step number of solution times
tec_step1 -- number of steps in TEC subrange (+/- last TEC fit value)
tec_step2 -- number of steps in full TEC range (-0.1 -- 0.1)
search_full_tec_range -- always search the full TEC range (-0.1 -- 0.1)
"""
from pylab import pinv, newaxis, find, amin
import numpy as np
from lofar.expion import baselinefitting
try:
import progressbar
except ImportError:
import losoto.progressbar as progressbar
N_sources_selected = len(source_selection)
N_stations_selected = len(station_selection)
N_piercepoints = N_sources_selected * N_stations_selected
N_times = len(times)
N_stations = len(station_names)
N_sources = len(source_names)
N_pairs = 0
for ii, i in enumerate(source_selection):
for jj, j in enumerate(source_selection):
if j == i:
break
subband_selection = find(mask[i,:] * mask[j,:])
if len(subband_selection) < nband_min:
continue
N_pairs += 1
D = np.resize(station_positions, (N_stations, N_stations, 3))
D = np.transpose(D, (1, 0, 2)) - D
D = np.sqrt(np.sum(D**2, axis=2))
station_selection1 = station_selection
stations_to_add = np.array([i for i in xrange(len(station_names))
if i not in station_selection1 and station_names[i] not in
excluded_stations])
if len(stations_to_add) == 0:
return station_selection1, r
# Check if desired number of stations is already reached
if nstations_max is not None:
if len(station_selection1) >= nstations_max:
return station_selection1, r
logging.info("Using fitting with iterative search for remaining stations "
"(up to {0} stations in total)".format(nstations_max))
q = r
while len(stations_to_add)>0:
D1 = D[stations_to_add[:,newaxis], station_selection1[newaxis,:]]
minimum_distance = amin(D1, axis=1)
station_to_add = stations_to_add[np.argmin(minimum_distance)]
station_selection1 = np.append(station_selection1, station_to_add)
N_stations_selected1 = len(station_selection1)
# Remove station from list
stations_to_add = stations_to_add[stations_to_add != station_to_add]
sols_list = []
eq_list = []
min_e_list = []
ipbar = 0
logging.info('Fitting TEC values with {0} included...'.format(
station_names[station_to_add]))
pbar = progressbar.ProgressBar(maxval=N_pairs*N_times).start()
for ii, i in enumerate(source_selection):
for jj, j in enumerate(source_selection):
if j == i:
break
subband_selection = find(mask[i,:] * mask[j,:])
if len(subband_selection) < nband_min:
continue
logging.debug('Adding {0} for source pair: {1}-{2}'.format(
station_names[station_to_add], i, j))
p0 = phases0[i, station_selection1[:,newaxis],
subband_selection[newaxis,:], :] - phases0[j,
station_selection1[:,newaxis], subband_selection[newaxis,:], :]
p0 = p0 - np.mean(p0, axis=0)[newaxis,:,:]
if soln_type != 'scalarphase':
p1 = phases1[i, station_selection1[:,newaxis],
subband_selection[newaxis,:], :] - phases1[j,
station_selection1[:,newaxis], subband_selection[newaxis,:], :]
p1 = p1 - np.mean(p1, axis=0)[newaxis, :, :]
A = np.zeros((len(subband_selection), 1))
A[:, 0] = 8.44797245e9 / freqs[subband_selection]
sd = (0.1 * 30e6) / freqs[subband_selection] # standard deviation of phase solutions as function of frequency (rad)
flags_source_pair = flags[i, station_selection1[:,newaxis],
subband_selection[newaxis,:], :] * flags[j,
station_selection1[:,newaxis], subband_selection[newaxis,:], :]
constant_parms = np.zeros((1, N_stations_selected1), dtype = np.bool)
sols = np.zeros((N_times, N_stations_selected1), dtype = np.float)
p_0_best = None
for t_idx in xrange(N_times):
if np.mod(t_idx, t_step) == 0:
min_e = np.Inf
if p_0_best is not None and not search_full_tec_range:
min_tec = p_0_best[0, -1] - 0.02
max_tec = p_0_best[0, -1] + 0.02
nsteps = tec_step1
else:
min_tec = -0.1
max_tec = 0.1
nsteps = tec_step2
logging.debug(' Trying initial guesses between {0} and '
'{1} TECU'.format(min_tec, max_tec))
for offset in np.linspace(min_tec, max_tec, nsteps):
p_0 = np.zeros((1, N_stations_selected1), np.double)
p_0[0, :N_stations_selected1-1] = (q[ii, t_idx, :] -
q[jj, t_idx, :])[newaxis,:]
p_0[0, -1] = offset
x = p0[:,:,t_idx].copy()
f = flags_source_pair[:, :, t_idx].copy()
sol0 = baselinefitting.fit(x.T, A, p_0, f, constant_parms)
sol0 -= np.mean(sol0)
residual = np.mod(np.dot(A, sol0) - x.T + np.pi,
2 * np.pi) - np.pi
e = np.var(residual[f.T==0])
if soln_type != 'scalarphase':
x = p1[:,:,t_idx].copy()
f = flags_source_pair[:, :, t_idx].copy()
sol1 = baselinefitting.fit(x.T, A, p_0, f,
constant_parms)
sol1 -= np.mean(sol1)
residual = np.mod(np.dot(A, sol1) - x.T + np.pi,
2 * np.pi) - np.pi
residual = residual[f.T==0]
e += np.var(residual)
else:
sol1 = sol0
if e < min_e:
logging.debug(' Found new min variance of {0} '
'with initial guess of {1} TECU'.format(e,
p_0[0, -1]))
min_e = e
p_0_best = p_0
sols[t_idx, :] = (sol0[0, :] + sol1[0, :])/2
else:
# Use previous init
x = p0[:, :, t_idx].copy()
f = flags_source_pair[:, :, t_idx].copy()
sol0 = baselinefitting.fit(x.T, A, p_0_best, f,
constant_parms)
sol0 -= np.mean(sol0)
if soln_type != 'scalarphase':
x = p1[:, :, t_idx].copy()
f = flags_source_pair[:, :, t_idx].copy()
sol1 = baselinefitting.fit(x.T, A, p_0_best, f,
constant_parms)
sol1 -= np.mean(sol1)
else:
sol1 = sol0
sols[t_idx, :] = (sol0[0, :] + sol1[0, :])/2
ipbar += 1
pbar.update(ipbar)
### Remove outliers
logging.debug(' Searching for outliers...')
for kk in xrange(10):
s = sols[:, -1].copy()
selection = np.zeros(len(s), np.bool)
for t_idx in xrange(len(s)):
start_idx = np.max([t_idx-10, 0])
end_idx = np.min([t_idx+10, len(s)])
selection[t_idx] = np.sum(abs(s[start_idx:end_idx] -
s[t_idx]) < 0.02) > (end_idx - start_idx - 8)
outliers = find(np.logical_not(selection))
if len(outliers) == 0:
break
for t_idx in outliers:
try:
idx0 = find(selection[:t_idx])[-1]
except IndexError:
idx0 = -1
try:
idx1 = find(selection[t_idx+1:])[0] + t_idx + 1
except IndexError:
idx1 = -1
if idx0 == -1:
s[t_idx] = s[idx1]
elif idx1 == -1:
s[t_idx] = s[idx0]
else:
s[t_idx] = (s[idx0] * (idx1-t_idx) + s[idx1] *
(t_idx-idx0)) / (idx1-idx0)
p_0 = np.zeros((1, N_stations_selected1), np.double)
p_0[0,:] = sols[t_idx,:]
p_0[0,-1] = s[t_idx]
x = p0[:,:,t_idx].copy()
f = flags_source_pair[:,:,t_idx].copy()
sol0 = baselinefitting.fit(x.T, A, p_0, f, constant_parms)
sol0 -= np.mean(sol0)
if soln_type != 'scalarphase':
x = p1[:,:,t_idx].copy()
sol1 = baselinefitting.fit(x.T, A, p_0, f,
constant_parms)
sol1 -= np.mean(sol1)
else:
sol1 = sol0
sols[t_idx, :] = (sol0[0,:] + sol1[0,:])/2
weight = 1.0
sols_list.append(weight*sols)
min_e_list.append(min_e)
eq = np.zeros(N_sources)
eq[ii] = weight
eq[jj] = -weight
eq_list.append(eq)
sols = np.array(sols_list)
B = np.array(eq_list)
pinvB = pinv(B)
q = np.dot(pinvB, sols.transpose([1,0,2]))
pbar.finish()
if nstations_max is not None:
if N_stations_selected1 == nstations_max:
break
return station_selection1, q
def fit_tec_per_source_pair(phases, flags, mask, freqs, init_sols=None,
init_sols_per_pair=False, propagate=False, nband_min=2):
"""Fits TEC values to phase solutions per source pair
Returns TEC solutions as array of shape (N_sources, N_times, N_stations)
Keyword arguments:
phases -- array of phase solutions
flags -- phase solution flags (0 = use, 1 = flagged)
mask -- mask for sources and frequencies (0 = ignore, 1 = use)
freqs -- array of frequencies
init_sols -- solutions to use to initialize the fits
init_sols_per_pair -- init_sols are per source pair (not per source)
propagate -- propagate solutions from previous solution
nband_min -- min number of bands for a source to be used
"""
from pylab import pinv, newaxis, find
import numpy as np
from lofar.expion import baselinefitting
try:
import progressbar
except ImportError:
import losoto.progressbar as progressbar
sols_list = []
eq_list = []
vars_list = []
var_eq_list = []
N_sources = phases.shape[0]
N_stations = phases.shape[1]
N_times = phases.shape[3]
N_pairs = 0
for i in xrange(N_sources):
for j in xrange(i):
subband_selection = find(mask[i, :] * mask[j, :])
if len(subband_selection) < nband_min:
continue
N_pairs += 1
if init_sols is None and not init_sols_per_pair:
init_sols = np.zeros((N_sources, N_times, N_stations), dtype = np.float)
elif init_sols is None and init_sols_per_pair:
init_sols = np.zeros((N_pairs, N_times, N_stations), dtype = np.float)
source_pairs = []
k = 0
ipbar = 0
logging.info('Fitting TEC values...')
pbar = progressbar.ProgressBar(maxval=N_pairs*N_times).start()
for i in xrange(N_sources):
for j in xrange(i):
subband_selection = find(mask[i, :] * mask[j, :])
if len(subband_selection) < nband_min:
continue
source_pairs.append((i, j))
p = phases[i, :, subband_selection, :] - phases[j, :, subband_selection, :]
A = np.zeros((len(subband_selection), 1))
A[:, 0] = 8.44797245e9/freqs[subband_selection]
flags_source_pair = flags[i, :, subband_selection, :] * flags[j, :,
subband_selection, :]
constant_parms = np.zeros((1, N_stations), dtype = np.bool)
sols = np.zeros((N_times, N_stations), dtype = np.float)
if init_sols_per_pair:
p_0 = init_sols[k, 0, :][newaxis, :]
else:
p_0 = (init_sols[i, 0, :] - init_sols[j, 0, :])[newaxis, :]
for t_idx in xrange(N_times):
x = p[:, :, t_idx].copy()
f = flags_source_pair[:, :, t_idx].copy()
if not propagate:
if init_sols_per_pair:
p_0 = init_sols[k, t_idx, :][newaxis, :]
else:
p_0 = (init_sols[i, t_idx, :] - init_sols[j, 0, :])[newaxis, :]
sol = baselinefitting.fit(x, A, p_0, f, constant_parms)
if propagate:
p_0 = sol.copy()
sols[t_idx, :] = sol[0, :]
ipbar += 1
pbar.update(ipbar)
sols = sols[:, :] - np.mean(sols[:, :], axis=1)[:, newaxis]
weight = len(subband_selection)
sols_list.append(sols*weight)
eq = np.zeros(N_sources)
eq[i] = weight
eq[j] = -weight
eq_list.append(eq)
k += 1
pbar.finish()
sols = np.array(sols_list)
B = np.array(eq_list)
source_selection = find(np.sum(abs(B), axis=0))
N_sources = len(source_selection)
if N_sources == 0:
logging.error('All sources have fewer than the required minimum number of bands.')
return None, None
pinvB = pinv(B)
r = np.dot(pinvB, sols.transpose([1, 0, 2]))
r = r[source_selection, :, :]
return r, source_selection
def add_stations(station_selection,
phases0,
phases1,
flags,
mask,
station_names,
station_positions,
source_names,
source_selection,
times,
freqs,
r,
nband_min=2,
soln_type='phase',
nstations_max=None,
excluded_stations=None,
t_step=5,
tec_step1=5,
tec_step2=21,
search_full_tec_range=False):
"""
Adds stations to TEC fitting using an iterative initial-guess search to
ensure the global min is found
Keyword arguments:
station_selection -- indices of stations to use in fitting
phases0 -- XX phase solutions
phases1 -- YY phase solutions
flags -- phase solution flags (0 = use, 1 = flagged)
mask -- mask for sources and frequencies (0 = ignore, 1 = use)
station_names -- array of station names
source_names -- array of source names
source_selection -- indices of sources to use in fitting
times -- array of times
freqs -- array of frequencies
r -- array of TEC solutions returned by fit_tec_per_source_pair()
nband_min -- min number of bands for a source to be used
soln_type -- type of phase solution: 'phase' or 'scalarphase'
nstations_max -- max number of stations to use
excluded_stations -- stations to exclude
t_step -- try full TEC range every t_step number of solution times
tec_step1 -- number of steps in TEC subrange (+/- last TEC fit value)
tec_step2 -- number of steps in full TEC range (-0.1 -- 0.1)
search_full_tec_range -- always search the full TEC range (-0.1 -- 0.1)
"""
from pylab import pinv, newaxis, find, amin
import numpy as np
from lofar.expion import baselinefitting
import progressbar
N_sources_selected = len(source_selection)
N_stations_selected = len(station_selection)
N_piercepoints = N_sources_selected * N_stations_selected
N_times = len(times)
N_stations = len(station_names)
N_sources = len(source_names)
N_pairs = 0
for ii, i in enumerate(source_selection):
for jj, j in enumerate(source_selection):
if j == i:
break
subband_selection = find(mask[i, :] * mask[j, :])
if len(subband_selection) < nband_min:
continue
N_pairs += 1
D = np.resize(station_positions, (N_stations, N_stations, 3))
D = np.transpose(D, (1, 0, 2)) - D
D = np.sqrt(np.sum(D**2, axis=2))
station_selection1 = station_selection
stations_to_add = np.array([
i for i in range(len(station_names)) if i not in station_selection1
and station_names[i] not in excluded_stations
])
if len(stations_to_add) == 0:
return station_selection1, r
# Check if desired number of stations is already reached
if nstations_max is not None:
if len(station_selection1) >= nstations_max:
return station_selection1, r
logging.info("Using fitting with iterative search for remaining stations "
"(up to {0} stations in total)".format(nstations_max))
q = r
while len(stations_to_add) > 0:
D1 = D[stations_to_add[:, newaxis], station_selection1[newaxis, :]]
minimum_distance = amin(D1, axis=1)
station_to_add = stations_to_add[np.argmin(minimum_distance)]
station_selection1 = np.append(station_selection1, station_to_add)
N_stations_selected1 = len(station_selection1)
# Remove station from list
stations_to_add = stations_to_add[stations_to_add != station_to_add]
sols_list = []
eq_list = []
min_e_list = []
ipbar = 0
logging.info('Fitting TEC values with {0} included...'.format(
station_names[station_to_add]))
pbar = progressbar.ProgressBar(maxval=N_pairs * N_times).start()
for ii, i in enumerate(source_selection):
for jj, j in enumerate(source_selection):
if j == i:
break
subband_selection = find(mask[i, :] * mask[j, :])
if len(subband_selection) < nband_min:
continue
logging.debug('Adding {0} for source pair: {1}-{2}'.format(
station_names[station_to_add], i, j))
p0 = phases0[i, station_selection1[:, newaxis],
subband_selection[newaxis, :], :] - phases0[
j, station_selection1[:, newaxis],
subband_selection[newaxis, :], :]
p0 = p0 - np.mean(p0, axis=0)[newaxis, :, :]
if soln_type != 'scalarphase':
p1 = phases1[i, station_selection1[:, newaxis],
subband_selection[newaxis, :], :] - phases1[
j, station_selection1[:, newaxis],
subband_selection[newaxis, :], :]
p1 = p1 - np.mean(p1, axis=0)[newaxis, :, :]
A = np.zeros((len(subband_selection), 1))
A[:, 0] = 8.44797245e9 / freqs[subband_selection]
flags_source_pair = flags[
i, station_selection1[:, newaxis],
subband_selection[newaxis, :], :] * flags[
j, station_selection1[:, newaxis],
subband_selection[newaxis, :], :]
constant_parms = np.zeros((1, N_stations_selected1),
dtype=np.bool)
sols = np.zeros((N_times, N_stations_selected1),
dtype=np.float)
p_0_best = None
for t_idx in range(N_times):
if np.mod(t_idx, t_step) == 0:
min_e = np.Inf
if p_0_best is not None and not search_full_tec_range:
min_tec = p_0_best[0, -1] - 0.02
max_tec = p_0_best[0, -1] + 0.02
nsteps = tec_step1
else:
min_tec = -0.1
max_tec = 0.1
nsteps = tec_step2
logging.debug(
' Trying initial guesses between {0} and '
'{1} TECU'.format(min_tec, max_tec))
for offset in np.linspace(min_tec, max_tec, nsteps):
p_0 = np.zeros((1, N_stations_selected1),
np.double)
p_0[0, :N_stations_selected1 -
1] = (q[ii, t_idx, :] -
q[jj, t_idx, :])[newaxis, :]
p_0[0, -1] = offset
x = p0[:, :, t_idx].copy()
f = flags_source_pair[:, :, t_idx].copy()
sol0 = baselinefitting.fit(x.T, A, p_0, f,
constant_parms)
sol0 -= np.mean(sol0)
residual = np.mod(
np.dot(A, sol0) - x.T + np.pi,
2 * np.pi) - np.pi
residual = residual[f.T == 0]
e = np.var(residual)
if soln_type != 'scalarphase':
x = p1[:, :, t_idx].copy()
f = flags_source_pair[:, :, t_idx].copy()
sol1 = baselinefitting.fit(
x.T, A, p_0, f, constant_parms)
sol1 -= np.mean(sol1)
residual = np.mod(
np.dot(A, sol1) - x.T + np.pi,
2 * np.pi) - np.pi
residual = residual[f.T == 0]
e += np.var(residual)
else:
sol1 = sol0
if e < min_e:
logging.debug(
' Found new min variance of {0} '
'with initial guess of {1} TECU'.format(
e, p_0[0, -1]))
min_e = e
p_0_best = p_0
sols[t_idx, :] = (sol0[0, :] + sol1[0, :]) / 2
else:
# Use previous init
x = p0[:, :, t_idx].copy()
f = flags_source_pair[:, :, t_idx].copy()
sol0 = baselinefitting.fit(x.T, A, p_0_best, f,
constant_parms)
sol0 -= np.mean(sol0)
if soln_type != 'scalarphase':
x = p1[:, :, t_idx].copy()
f = flags_source_pair[:, :, t_idx].copy()
sol1 = baselinefitting.fit(x.T, A, p_0_best, f,
constant_parms)
sol1 -= np.mean(sol1)
else:
sol1 = sol0
sols[t_idx, :] = (sol0[0, :] + sol1[0, :]) / 2
ipbar += 1
pbar.update(ipbar)
### Remove outliers
logging.debug(' Searching for outliers...')
for kk in range(10):
s = sols[:, -1].copy()
selection = np.zeros(len(s), np.bool)
for t_idx in range(len(s)):
start_idx = np.max([t_idx - 10, 0])
end_idx = np.min([t_idx + 10, len(s)])
selection[t_idx] = np.sum(
abs(s[start_idx:end_idx] - s[t_idx]) < 0.02) > (
end_idx - start_idx - 8)
outliers = find(np.logical_not(selection))
if len(outliers) == 0:
break
for t_idx in outliers:
try:
idx0 = find(selection[:t_idx])[-1]
except IndexError:
idx0 = -1
try:
idx1 = find(selection[t_idx + 1:])[0] + t_idx + 1
except IndexError:
idx1 = -1
if idx0 == -1:
s[t_idx] = s[idx1]
elif idx1 == -1:
s[t_idx] = s[idx0]
else:
s[t_idx] = (s[idx0] * (idx1 - t_idx) + s[idx1] *
(t_idx - idx0)) / (idx1 - idx0)
p_0 = np.zeros((1, N_stations_selected1), np.double)
p_0[0, :] = sols[t_idx, :]
p_0[0, -1] = s[t_idx]
x = p0[:, :, t_idx].copy()
f = flags_source_pair[:, :, t_idx].copy()
sol0 = baselinefitting.fit(x.T, A, p_0, f,
constant_parms)
sol0 -= np.mean(sol0)
if soln_type != 'scalarphase':
x = p1[:, :, t_idx].copy()
sol1 = baselinefitting.fit(x.T, A, p_0, f,
constant_parms)
sol1 -= np.mean(sol1)
else:
sol1 = sol0
sols[t_idx, :] = (sol0[0, :] + sol1[0, :]) / 2
weight = 1.0
sols_list.append(weight * sols)
min_e_list.append(min_e)
eq = np.zeros(N_sources)
eq[ii] = weight
eq[jj] = -weight
eq_list.append(eq)
sols = np.array(sols_list)
B = np.array(eq_list)
pinvB = pinv(B)
q = np.dot(pinvB, sols.transpose([1, 0, 2]))
pbar.finish()
if nstations_max is not None:
if N_stations_selected1 == nstations_max:
break
return station_selection1, q
def fit_tec_per_source_pair(phases,
flags,
mask,
freqs,
init_sols=None,
init_sols_per_pair=False,
propagate=False,
nband_min=2):
"""Fits TEC values to phase solutions per source pair
Returns TEC solutions as array of shape (N_sources, N_times, N_stations)
Keyword arguments:
phases -- array of phase solutions
flags -- phase solution flags (0 = use, 1 = flagged)
mask -- mask for sources and frequencies (0 = ignore, 1 = use)
freqs -- array of frequencies
init_sols -- solutions to use to initialize the fits
init_sols_per_pair -- init_sols are per source pair (not per source)
propagate -- propagate solutions from previous solution
nband_min -- min number of bands for a source to be used
"""
from pylab import pinv, newaxis, find
import numpy as np
from lofar.expion import baselinefitting
import progressbar
sols_list = []
eq_list = []
vars_list = []
var_eq_list = []
N_sources = phases.shape[0]
N_stations = phases.shape[1]
N_times = phases.shape[3]
N_pairs = 0
for i in range(N_sources):
for j in range(i):
subband_selection = find(mask[i, :] * mask[j, :])
if len(subband_selection) < nband_min:
continue
N_pairs += 1
if init_sols is None and not init_sols_per_pair:
init_sols = np.zeros((N_sources, N_times, N_stations), dtype=np.float)
elif init_sols is None and init_sols_per_pair:
init_sols = np.zeros((N_pairs, N_times, N_stations), dtype=np.float)
source_pairs = []
k = 0
ipbar = 0
logging.info('Fitting TEC values...')
pbar = progressbar.ProgressBar(maxval=N_pairs * N_times).start()
for i in range(N_sources):
for j in range(i):
subband_selection = find(mask[i, :] * mask[j, :])
if len(subband_selection) < nband_min:
continue
source_pairs.append((i, j))
p = phases[i, :,
subband_selection, :] - phases[j, :,
subband_selection, :]
A = np.zeros((len(subband_selection), 1))
A[:, 0] = 8.44797245e9 / freqs[subband_selection]
flags_source_pair = flags[i, :, subband_selection, :] * flags[
j, :, subband_selection, :]
constant_parms = np.zeros((1, N_stations), dtype=np.bool)
sols = np.zeros((N_times, N_stations), dtype=np.float)
if init_sols_per_pair:
p_0 = init_sols[k, 0, :][newaxis, :]
else:
p_0 = (init_sols[i, 0, :] - init_sols[j, 0, :])[newaxis, :]
for t_idx in range(N_times):
x = p[:, :, t_idx].copy()
f = flags_source_pair[:, :, t_idx].copy()
if not propagate:
if init_sols_per_pair:
p_0 = init_sols[k, t_idx, :][newaxis, :]
else:
p_0 = (init_sols[i, t_idx, :] -
init_sols[j, 0, :])[newaxis, :]
sol = baselinefitting.fit(x, A, p_0, f, constant_parms)
if propagate:
p_0 = sol.copy()
sols[t_idx, :] = sol[0, :]
ipbar += 1
pbar.update(ipbar)
sols = sols[:, :] - np.mean(sols[:, :], axis=1)[:, newaxis]
weight = len(subband_selection)
sols_list.append(sols * weight)
eq = np.zeros(N_sources)
eq[i] = weight
eq[j] = -weight
eq_list.append(eq)
k += 1
pbar.finish()
sols = np.array(sols_list)
B = np.array(eq_list)
source_selection = find(np.sum(abs(B), axis=0))
N_sources = len(source_selection)
if N_sources == 0:
logging.error(
'All sources have fewer than the required minimum number of bands.'
)
return None, None
pinvB = pinv(B)
r = np.dot(pinvB, sols.transpose([1, 0, 2]))
r = r[source_selection, :, :]
return r, source_selection
def getClockTECBaselineFit(ph,amp,freqs,SBselect,polIdx,stIdx,useOffset=False,stations=[],initSol=[],chi2cut=300.,fixedClockforCS=False,timeIdx=0):
global tecarray
global clockarray
global offsetarray
global residualarray
amp[np.isnan(ph)]=1
ph[np.isnan(ph)]=0.
ph=np.unwrap(ph,axis=0)
#first unwrap data and get initial guess
nT=ph.shape[0]
nF=freqs.shape[0]
nSt=ph.shape[2]
nparms=2+(useOffset>0)
#sol = np.zeros((nSt,nparms),dtype=np.float)
sol = np.zeros((nSt,nparms),dtype=np.float)
print sol.shape,nparms,nSt
A=np.zeros((nF,nparms),dtype=np.float)
A[:,1] = freqs*2*np.pi*(-1e-9)
A[:,0] = -8.44797245e9/freqs
if useOffset:
A[:,2] = np.ones((nF,))
constant_parms=np.zeros(sol.shape,dtype=bool)
if fixedClockforCS:
for ist,st in enumerate(stations):
if 'CS' in st:
constant_parms[ist,1]=True
stepDelay=1
if "HBA" in stations[0]:
stepdTEC=0.005
else:
stepdTEC=0.001 # faster wrapping for LBA
stepDelay=3
succes=False
initprevsol=False
nrFail=0
for itm in range(nT):
if itm%100==0 and itm>0:
sys.stdout.write(str(itm)+'... '+str(sol[-1,0]-sol[0,0])+' '+str(sol[-1,1]-sol[0,1])+' '+str(sol[-1,-1]-sol[0,-1])+' ')
sys.stdout.flush()
flags=(amp[itm,:]==1);
nrFlags=np.sum(flags,axis=0)
if itm==0 or not succes:
for ist in range(1,nSt):
if (nF-nrFlags[ist])<10:
print "Too many data points flagged",itm,ist
continue;
if itm==0 or not initprevsol:
if hasattr(initSol,'__len__') and len(initSol)>ist:
iTEC1=initSol[ist,0]
iTEC2=initSol[ist,0]+stepdTEC
iD1=initSol[ist,1]
iD2=initSol[ist,1]+stepDelay
else:
if 'CS' in stations[ist]:
iTEC1=-0.2
iTEC2=0.2
iD1=-4
iD2=4
else:
iTEC1=-1.5
iTEC2=1.5
iD1=-50
iD2=300
print "First",iTEC1,iTEC2,iD1,iD2
else:
iTEC1=prevsol[ist,0]-stepdTEC*nrFail
iTEC2=prevsol[ist,0]+stepdTEC*(nrFail+1)
if not fixedClockforCS or not 'CS' in stations[ist]:
iD1=prevsol[ist,1]-stepDelay*nrFail
iD2=prevsol[ist,1]+stepDelay*(nrFail+1)
else:
iD1=sol[ist,1]
iD2=sol[ist,1]+stepDelay
print "Failure",iTEC1,iTEC2,iD1,iD2,nrFail
dTECArray=np.arange(iTEC1,iTEC2,stepdTEC)
dClockArray=np.arange(iD1,iD2,stepDelay)
data=ph[itm,:,ist][np.logical_not(np.logical_or(flags[:,ist],flags[:,0]))]-ph[itm,:,0][np.logical_not(np.logical_or(flags[:,ist],flags[:,0]))]
tmpfreqs=freqs[np.logical_not(np.logical_or(flags[:,ist],flags[:,0]))]
print "getting init",ist,
par = getInitPar(data,dTECArray, dClockArray,tmpfreqs,ClockTECfunc)
print par
sol[ist,:]=par[:nparms]
if not succes:
#reset first station
sol[0,:] = np.zeros(nparms)
#sol=fitting.fit(ph[itm],A,sol.T,flags).T
#sol=fitting.fit(ph[itm],A,sol,flags)
sol=fitting.fit(ph[itm],A,sol.T,flags,constant_parms.T).T
tecarray[itm+timeIdx,stIdx,polIdx]=sol[:,0]
clockarray[itm+timeIdx,stIdx,polIdx]=sol[:,1]
if useOffset:
offsetarray[itm+timeIdx,stIdx,polIdx]+=sol[:,2]
residual = ph[itm] - np.dot(A, sol.T)
residual = residual - residual[:, 0][:,np.newaxis]
residual = np.remainder(residual+np.pi, 2*np.pi) - np.pi
residual[flags]=0
residualarray[np.ix_([itm+timeIdx],SBselect,stIdx,[polIdx])]=residual.reshape((1,nF,nSt,1))
chi2=np.sum(np.square(np.degrees(residual)))/(nSt*nF)
if chi2>chi2cut:
print "failure",chi2,sol
succes=False
nrFail=0
else:
prevsol=np.copy(sol)
# print "succes",chi2,dTECArray.shape,dClockArray.shape
succes=True
initprevsol=True
nrFail+=1
def add_stations(station_selection, phases0, phases1, flags, mask, station_names, station_positions, source_names, source_selection, times, freqs, r, nband_min=2, soln_type='phase', nstations_max=None):
N_sources_selected = len(source_selection)
N_stations_selected = len(station_selection)
N_piercepoints = N_sources_selected * N_stations_selected
N_times = len(times)
N_stations = len(station_names)
N_sources = len(source_names)
D = resize( station_positions, ( N_stations, N_stations, 3 ) )
D = transpose( D, ( 1, 0, 2 ) ) - D
D = sqrt(sum( D**2, axis=2 ))
station_selection1 = station_selection
stations_to_add = array([i for i in range(len(station_names)) if i not in station_selection1])
# Check if desired number of stations is already reached
if nstations_max is not None:
if len(station_selection1) >= nstations_max:
return station_selection1, None, r
q = r
while len(stations_to_add)>0:
D1 = D[stations_to_add[:,newaxis], station_selection1[newaxis,:]]
minimum_distance = amin(D1, axis=1)
station_to_add = stations_to_add[argmin(minimum_distance)]
station_selection1 = append(station_selection1, station_to_add)
N_stations_selected1 = len(station_selection1)
# Remove station from list
stations_to_add = stations_to_add[stations_to_add != station_to_add]
sols_list = []
eq_list = []
min_e_list = []
for ii, i in enumerate(source_selection):
for jj, j in enumerate(source_selection):
if j == i:
break
subband_selection = find(mask[i,:] * mask[j,:])
if len(subband_selection) < nband_min:
continue
logging.info('Fitting {0} TEC values for source pair: {1}-{2}'.format(station_names[station_to_add], i, j))
p0 = phases0[i, station_selection1[:,newaxis], subband_selection[newaxis,:], :] - phases0[j, station_selection1[:,newaxis], subband_selection[newaxis,:], :]
p0 = p0 - mean(p0, axis=0)[newaxis,:,:]
if soln_type != 'scalarphase':
p1 = phases1[i, station_selection1[:,newaxis], subband_selection[newaxis,:], :] - phases1[j, station_selection1[:,newaxis], subband_selection[newaxis,:], :]
p1 = p1 - mean(p1, axis=0)[newaxis,:,:]
A = zeros((len(subband_selection), 1))
A[:,0] = 8.44797245e9/freqs[subband_selection]
flags_source_pair = flags[i, station_selection1[:,newaxis], subband_selection[newaxis,:], :] * flags[j, station_selection1[:,newaxis], subband_selection[newaxis,:], :]
constant_parms = zeros((1, N_stations_selected1), dtype = bool)
sols = zeros((N_times, N_stations_selected1), dtype = float)
for t_idx in range(N_times):
if np.mod(t_idx, 10) == 0:
min_e = Inf
for offset in linspace(-0.1, 0.1,21) :
p_0 = zeros((1, N_stations_selected1), double)
p_0[0,:N_stations_selected1-1] = (q[ii, t_idx, :] - q[jj, t_idx, :])[newaxis,:]
p_0[0,-1] = offset
x = p0[:,:,t_idx].copy()
f = flags_source_pair[:,:,t_idx].copy()
sol0 = baselinefitting.fit(x.T, A, p_0, f, constant_parms)
sol0 -= mean(sol0)
residual = mod(dot(A,sol0) - x.T + pi, 2*pi) - pi
residual = residual[f.T==0]
e = var(residual)
if soln_type != 'scalarphase':
x = p1[:,:,t_idx].copy()
f = flags_source_pair[:,:,t_idx].copy()
sol1 = baselinefitting.fit(x.T, A, p_0, f, constant_parms)
sol1 -= mean(sol1)
residual = mod(dot(A,sol1) - x.T + pi, 2*pi) - pi
residual = residual[f.T==0]
e += var(residual)
else:
sol1 = sol0
if e<min_e:
min_e = e
p_0_best = p_0
sols[t_idx, :] = (sol0[0,:] + sol1[0,:])/2
else:
# Use previous init
x = p0[:,:,t_idx].copy()
f = flags_source_pair[:,:,t_idx].copy()
sol0 = baselinefitting.fit(x.T, A, p_0_best, f, constant_parms)
sol0 -= mean(sol0)
if soln_type != 'scalarphase':
x = p1[:,:,t_idx].copy()
f = flags_source_pair[:,:,t_idx].copy()
sol1 = baselinefitting.fit(x.T, A, p_0_best, f, constant_parms)
sol1 -= mean(sol1)
else:
sol1 = sol0
sols[t_idx, :] = (sol0[0,:] + sol1[0,:])/2
### Remove outliers
for kk in range(10):
s = sols[:,-1].copy()
selection = zeros(len(s), bool)
for t_idx in range(len(s)):
start_idx = max(t_idx-10, 0)
end_idx = min(t_idx+10, len(s))
selection[t_idx] = sum(abs(s[start_idx:end_idx] - s[t_idx])<0.02) > (end_idx - start_idx - 8)
outliers = find(logical_not(selection))
if len(outliers) == 0:
break
for t_idx in outliers:
try:
idx0 = find(selection[:t_idx])[-1]
except IndexError:
idx0 = -1
try:
idx1 = find(selection[t_idx+1:])[0]+t_idx+1
except IndexError:
idx1 = -1
if idx0 == -1:
s[t_idx] = s[idx1]
elif idx1 == -1:
s[t_idx] = s[idx0]
else:
s[t_idx] = (s[idx0] * (idx1-t_idx) + s[idx1] * (t_idx-idx0)) / (idx1-idx0)
p_0 = zeros((1, N_stations_selected1), double)
p_0[0,:] = sols[t_idx,:]
p_0[0,-1] = s[t_idx]
x = p0[:,:,t_idx].copy()
f = flags_source_pair[:,:,t_idx].copy()
sol0 = baselinefitting.fit(x.T, A, p_0, f, constant_parms)
sol0 -= mean(sol0)
if soln_type != 'scalarphase':
x = p1[:,:,t_idx].copy()
sol1 = baselinefitting.fit(x.T, A, p_0, f, constant_parms)
sol1 -= mean(sol1)
else:
sol1 = sol0
sols[t_idx, :] = (sol0[0,:] + sol1[0,:])/2
weight = 1.0
sols_list.append(weight*sols)
min_e_list.append(min_e)
eq = zeros(N_sources)
eq[ii] = weight
eq[jj] = -weight
eq_list.append(eq)
sols = array(sols_list)
B = array(eq_list)
pinvB = pinv(B)
q = dot(pinvB, sols.transpose([1,0,2]))
if nstations_max is not None:
if N_stations_selected1 == nstations_max:
break
return station_selection1, sols, q
def fit_tec_per_source_pair(phases, flags, mask, freqs, init_sols=None,
init_sols_per_pair=False, propagate=False, nband_min=2, offset=0):
sols_list = []
eq_list = []
vars_list = []
var_eq_list = []
N_sources = phases.shape[0]
N_stations = phases.shape[1]
N_times = phases.shape[3]
k = 0
for i in range(N_sources):
for j in range(i):
k += 1
N_pairs = k
if init_sols is None and not init_sols_per_pair:
init_sols = zeros((N_sources, N_times, N_stations), dtype = float) + offset
elif init_sols is None and init_sols_per_pair:
init_sols = zeros((N_pairs, N_times, N_stations), dtype = float) + offset
vars = zeros((N_times, N_stations), dtype = float)
source_pairs = []
k = 0
for i in range(N_sources):
for j in range(i):
subband_selection = find(mask[i,:] * mask[j,:])
if len(subband_selection) < nband_min:
continue
logging.info('Fitting TEC values for source pair: {0}-{1}'.format(i, j))
source_pairs.append((i,j))
p = phases[i, :, subband_selection, :] - phases[j, :, subband_selection, :]
A = zeros((len(subband_selection), 1))
A[:,0] = 8.44797245e9/freqs[subband_selection]
flags_source_pair = flags[i, :, subband_selection, :] * flags[j, :, subband_selection, :]
constant_parms = zeros((1, N_stations), dtype = bool)
sols = zeros((N_times, N_stations), dtype = float)
if init_sols_per_pair:
p_0 = init_sols[k, 0, :][newaxis,:]
else:
p_0 = (init_sols[i, 0, :] - init_sols[j, 0, :])[newaxis,:]
for t_idx in range(N_times):
x = p[:,:,t_idx].copy()
f = flags_source_pair[:,:,t_idx].copy()
if not propagate:
if init_sols_per_pair:
p_0 = init_sols[k, t_idx, :][newaxis,:]
else:
p_0 = (init_sols[i, t_idx, :] - init_sols[j, 0, :])[newaxis,:]
sol = baselinefitting.fit(x, A, p_0, f, constant_parms)
if propagate:
p_0 = sol.copy()
sols[t_idx, :] = sol[0, :]
sols = sols[:, :]-mean(sols[:, :], axis=1)[:,newaxis]
weight = len(subband_selection)
sols_list.append(sols*weight)
vars_list.append(vars*weight)
eq = zeros(N_sources)
eq[i] = weight
eq[j] = -weight
eq_list.append(eq)
var_eq = zeros(N_sources)
var_eq[i] = weight
var_eq[j] = weight
var_eq_list.append(var_eq)
k += 1
sols = array(sols_list)
B = array(eq_list)
source_selection = find(sum(abs(B), axis=0))
N_sources = len(source_selection)
if N_sources == 0:
logging.error('No sources with enough bands.')
return None, None, None, None
pinvB = pinv(B)
r = dot(pinvB, sols.transpose([1,0,2]))
r = r[source_selection, :, :]
return r, source_selection, sols, source_pairs
def getClockTECBaselineFit(ph,
amp,
freqs,
SBselect,
polIdx,
stIdx,
useOffset=False,
stations=[],
initSol=[],
chi2cut=300.,
fixedClockforCS=False,
timeIdx=0):
global tecarray
global clockarray
global offsetarray
global residualarray
amp[np.isnan(ph)] = 1
ph[np.isnan(ph)] = 0.
ph = np.unwrap(ph, axis=0)
#first unwrap data and get initial guess
nT = ph.shape[0]
nF = freqs.shape[0]
nSt = ph.shape[2]
nparms = 2 + (useOffset > 0)
#sol = np.zeros((nSt,nparms),dtype=np.float)
sol = np.zeros((nSt, nparms), dtype=np.float)
print(sol.shape, nparms, nSt)
A = np.zeros((nF, nparms), dtype=np.float)
A[:, 1] = freqs * 2 * np.pi * (-1e-9)
A[:, 0] = -8.44797245e9 / freqs
if useOffset:
A[:, 2] = np.ones((nF, ))
constant_parms = np.zeros(sol.shape, dtype=bool)
if fixedClockforCS:
for ist, st in enumerate(stations):
if 'CS' in st:
constant_parms[ist, 1] = True
stepDelay = 1
if "HBA" in stations[0]:
stepdTEC = 0.005
else:
stepdTEC = 0.001 # faster wrapping for LBA
stepDelay = 3
succes = False
initprevsol = False
nrFail = 0
for itm in range(nT):
if itm % 100 == 0 and itm > 0:
sys.stdout.write(
str(itm) + '... ' + str(sol[-1, 0] - sol[0, 0]) + ' ' +
str(sol[-1, 1] - sol[0, 1]) + ' ' +
str(sol[-1, -1] - sol[0, -1]) + ' ')
sys.stdout.flush()
flags = (amp[itm, :] == 1)
nrFlags = np.sum(flags, axis=0)
if itm == 0 or not succes:
for ist in range(1, nSt):
if (nF - nrFlags[ist]) < 10:
print("Too many data points flagged", itm, ist)
continue
if itm == 0 or not initprevsol:
if hasattr(initSol, '__len__') and len(initSol) > ist:
iTEC1 = initSol[ist, 0]
iTEC2 = initSol[ist, 0] + stepdTEC
iD1 = initSol[ist, 1]
iD2 = initSol[ist, 1] + stepDelay
else:
if 'CS' in stations[ist]:
iTEC1 = -0.2
iTEC2 = 0.2
iD1 = -4
iD2 = 4
else:
iTEC1 = -1.5
iTEC2 = 1.5
iD1 = -50
iD2 = 300
print("First", iTEC1, iTEC2, iD1, iD2)
else:
iTEC1 = prevsol[ist, 0] - stepdTEC * nrFail
iTEC2 = prevsol[ist, 0] + stepdTEC * (nrFail + 1)
if not fixedClockforCS or not 'CS' in stations[ist]:
iD1 = prevsol[ist, 1] - stepDelay * nrFail
iD2 = prevsol[ist, 1] + stepDelay * (nrFail + 1)
else:
iD1 = sol[ist, 1]
iD2 = sol[ist, 1] + stepDelay
print("Failure", iTEC1, iTEC2, iD1, iD2, nrFail)
dTECArray = np.arange(iTEC1, iTEC2, stepdTEC)
dClockArray = np.arange(iD1, iD2, stepDelay)
data = ph[itm, :, ist][np.logical_not(
np.logical_or(
flags[:, ist],
flags[:, 0]))] - ph[itm, :, 0][np.logical_not(
np.logical_or(flags[:, ist], flags[:, 0]))]
tmpfreqs = freqs[np.logical_not(
np.logical_or(flags[:, ist], flags[:, 0]))]
print("getting init", ist, end=' ')
par = getInitPar(data, dTECArray, dClockArray, tmpfreqs,
ClockTECfunc)
print(par)
sol[ist, :] = par[:nparms]
if not succes:
#reset first station
sol[0, :] = np.zeros(nparms)
#sol=fitting.fit(ph[itm],A,sol.T,flags).T
#sol=fitting.fit(ph[itm],A,sol,flags)
sol = fitting.fit(ph[itm], A, sol.T, flags, constant_parms.T).T
tecarray[itm + timeIdx, stIdx, polIdx] = sol[:, 0]
clockarray[itm + timeIdx, stIdx, polIdx] = sol[:, 1]
if useOffset:
offsetarray[itm + timeIdx, stIdx, polIdx] += sol[:, 2]
residual = ph[itm] - np.dot(A, sol.T)
residual = residual - residual[:, 0][:, np.newaxis]
residual = np.remainder(residual + np.pi, 2 * np.pi) - np.pi
residual[flags] = 0
residualarray[np.ix_([itm + timeIdx], SBselect, stIdx,
[polIdx])] = residual.reshape((1, nF, nSt, 1))
chi2 = np.sum(np.square(np.degrees(residual))) / (nSt * nF)
if chi2 > chi2cut:
print("failure", chi2, sol)
succes = False
nrFail = 0
else:
prevsol = np.copy(sol)
# print "succes",chi2,dTECArray.shape,dClockArray.shape
succes = True
initprevsol = True
nrFail += 1
