def getTECBaselineFit(ph, amp, freqs, SBselect, polIdx, stIdx, useOffset=False, stations=[], initSol=[], chi2cut=300., timeIdx=0): global tecarray global offsetarray global residualarray amp[np.isnan(ph)] = 1 ph[np.isnan(ph)] = 0. ph = np.unwrap(ph, axis=0) #first unwrap data and get initial guess nT = ph.shape[0] nF = freqs.shape[0] nSt = ph.shape[2] nparms = 1 + (useOffset > 0) sol = np.zeros((nSt, nparms), dtype=np.float) A = np.zeros((nF, nparms), dtype=np.float) A[:, 0] = -8.44797245e9 / freqs if useOffset: A[:, 1] = np.ones((nF, )) # init first sol big_array = (np.arange(-0.1, 0.1, 0.005) * A[:, 0][:, np.newaxis]).T diff = np.sum(np.absolute( np.remainder( big_array[:, :, np.newaxis] - (ph[0, :, :] - ph[0, :, :][:, [0]]) + np.pi, 2 * np.pi) - np.pi), axis=1) init_idx = np.argmin(diff, axis=0) sol[:, 0] = init_idx * 0.005 - 0.1 print("Initializing with", sol[:, 0]) for itm in range(nT): if itm % 100 == 0 and itm > 0: sys.stdout.write( str(itm) + '... ' + str(sol[-1, 0] - sol[0, 0]) + ' ' + str(sol[-1, -1] - sol[0, -1]) + ' ') sys.stdout.flush() flags = (amp[itm, :] == 1) nrFlags = np.sum(flags, axis=0) sol = fitting.fit(ph[itm], A, sol.T, flags).T tecarray[itm + timeIdx, stIdx, polIdx] = sol[:, 0] if useOffset: offsetarray[itm + timeIdx, stIdx, polIdx] = sol[:, 1] residual = ph[itm] - np.dot(A, sol.T) residual = residual - residual[:, 0][:, np.newaxis] residual = np.remainder(residual + np.pi, 2 * np.pi) - np.pi residual[flags] = 0 residualarray[np.ix_([itm + timeIdx], SBselect, stIdx, [polIdx])] = residual.reshape((1, nF, nSt, 1))
def getResidualPhaseWraps(avgResiduals, freqs): flags = np.average(avgResiduals.mask, axis=1) > 0.5 nSt = avgResiduals.shape[1] nF = freqs.shape[0] wraps = np.zeros((nSt, ), dtype=np.float) tmpflags = flags tmpfreqs = freqs[np.logical_not(tmpflags)] (tmpbasef, steps) = getPhaseWrapBase(tmpfreqs) basef = np.zeros(freqs.shape) basef[np.logical_not(tmpflags)] = tmpbasef basef = basef.reshape((-1, 1)) data = avgResiduals[:, :] wraps = fitting.fit(data, basef, wraps, flags).flatten() return (wraps, steps)
def getResidualPhaseWraps(avgResiduals, freqs): flags = np.average(avgResiduals.mask, axis=1) > 0.5 nSt = avgResiduals.shape[1] nF = freqs.shape[0] wraps = np.zeros((nSt,), dtype=np.float) tmpflags = flags tmpfreqs = freqs[np.logical_not(tmpflags)] (tmpbasef, steps) = getPhaseWrapBase(tmpfreqs) basef = np.zeros(freqs.shape) basef[np.logical_not(tmpflags)] = tmpbasef basef = basef.reshape((-1, 1)) data = avgResiduals[:, :] wraps = fitting.fit(data, basef, wraps, flags).flatten() return (wraps, steps)
def getResidualPhaseWraps2(avgResiduals, freqs): flags = avgResiduals[:, 10] == 0. nSt = avgResiduals.shape[1] nF = freqs.shape[0] wraps = np.zeros((nSt, ), dtype=np.float) #tmpflags=np.sum(flags[:,np.sum(flags,axis=0)<(nF*0.5)],axis=1) tmpflags = flags tmpfreqs = freqs[np.logical_not(tmpflags)] tmpbasef, steps = getPhaseWrapBase(tmpfreqs) basef = np.zeros(freqs.shape) basef[np.logical_not(tmpflags)] = tmpbasef basef = basef.reshape((-1, 1)) data = avgResiduals[:, :] wraps = fitting.fit(data, basef, wraps, flags).flatten() return wraps, steps
def getResidualPhaseWraps(avgResiduals,freqs): flags=avgResiduals[:,10]==0. nSt=avgResiduals.shape[1] nF=freqs.shape[0] wraps=np.zeros((nSt,),dtype=np.float) #tmpflags=np.sum(flags[:,np.sum(flags,axis=0)<(nF*0.5)],axis=1) tmpflags=flags tmpfreqs=freqs[np.logical_not(tmpflags)] tmpbasef,steps=getPhaseWrapBase(tmpfreqs) basef=np.zeros(freqs.shape) basef[np.logical_not(tmpflags)]=tmpbasef basef=basef.reshape((-1,1)) data=avgResiduals[:,:] wraps=fitting.fit(data,basef,wraps,flags).flatten() return wraps,steps
def getResidualPhaseWraps(avgResiduals, freqs): flags = np.average(avgResiduals.mask,axis=1)>0.5 nSt = avgResiduals.shape[1] nF = freqs.shape[0] wraps = np.zeros((nSt, ), dtype=np.float) tmpflags = flags tmpfreqs = freqs[np.logical_not(tmpflags)] steps=[0,0] if np.ma.count(tmpfreqs) < 3: logging.debug('Cannot unwrap, too many channels flagged') return (wraps,steps) (tmpbasef, steps) = getPhaseWrapBase(tmpfreqs) basef = np.zeros(freqs.shape) basef[np.logical_not(tmpflags)] = tmpbasef basef = basef.reshape((-1, 1)) data = avgResiduals[:, :] if has_fitting: wraps = fitting.fit(data, basef, wraps, flags).flatten() return (wraps, steps)
def getTECBaselineFit(ph,amp,freqs,SBselect,polIdx,stIdx,useOffset=False,stations=[],initSol=[],chi2cut=300.,timeIdx=0): global tecarray global offsetarray global residualarray amp[np.isnan(ph)]=1 ph[np.isnan(ph)]=0. ph=np.unwrap(ph,axis=0) #first unwrap data and get initial guess nT=ph.shape[0] nF=freqs.shape[0] nSt=ph.shape[2] nparms=1+(useOffset>0) sol = np.zeros((nSt,nparms),dtype=np.float) A=np.zeros((nF,nparms),dtype=np.float) A[:,0] = -8.44797245e9/freqs if useOffset: A[:,1] = np.ones((nF,)) # init first sol big_array=(np.arange(-0.1,0.1,0.005)*A[:,0][:,np.newaxis]).T diff=np.sum(np.absolute(np.remainder(big_array[:,:,np.newaxis]-(ph[0,:,:]-ph[0,:,:][:,[0]])+np.pi,2*np.pi)-np.pi),axis=1) init_idx=np.argmin(diff,axis=0) sol[:,0]=init_idx*0.005-0.1 print "Initializing with",sol[:,0] for itm in range(nT): if itm%100==0 and itm>0: sys.stdout.write(str(itm)+'... '+str(sol[-1,0]-sol[0,0])+' '+str(sol[-1,-1]-sol[0,-1])+' ') sys.stdout.flush() flags=(amp[itm,:]==1); nrFlags=np.sum(flags,axis=0) sol=fitting.fit(ph[itm],A,sol.T,flags).T tecarray[itm+timeIdx,stIdx,polIdx]=sol[:,0] if useOffset: offsetarray[itm+timeIdx,stIdx,polIdx]=sol[:,1] residual = ph[itm] - np.dot(A, sol.T) residual = residual - residual[:, 0][:,np.newaxis] residual = np.remainder(residual+np.pi, 2*np.pi) - np.pi residual[flags]=0 residualarray[np.ix_([itm+timeIdx],SBselect,stIdx,[polIdx])]=residual.reshape((1,nF,nSt,1))
def add_stations(station_selection, phases0, phases1, flags, mask, station_names, station_positions, source_names, source_selection, times, freqs, r, nband_min=2, soln_type='phase', nstations_max=None, excluded_stations=None, t_step=5, tec_step1=5, tec_step2=21, search_full_tec_range=True): """ Adds stations to TEC fitting using an iterative initial-guess search to ensure the global min is found Keyword arguments: station_selection -- indices of stations to use in fitting phases0 -- XX phase solutions phases1 -- YY phase solutions flags -- phase solution flags (0 = use, 1 = flagged) mask -- mask for sources and frequencies (0 = ignore, 1 = use) station_names -- array of station names source_names -- array of source names source_selection -- indices of sources to use in fitting times -- array of times freqs -- array of frequencies r -- array of TEC solutions returned by fit_tec_per_source_pair() nband_min -- min number of bands for a source to be used soln_type -- type of phase solution: 'phase' or 'scalarphase' nstations_max -- max number of stations to use excluded_stations -- stations to exclude t_step -- try full TEC range every t_step number of solution times tec_step1 -- number of steps in TEC subrange (+/- last TEC fit value) tec_step2 -- number of steps in full TEC range (-0.1 -- 0.1) search_full_tec_range -- always search the full TEC range (-0.1 -- 0.1) """ from pylab import pinv, newaxis, find, amin import numpy as np from lofar.expion import baselinefitting try: import progressbar except ImportError: import losoto.progressbar as progressbar N_sources_selected = len(source_selection) N_stations_selected = len(station_selection) N_piercepoints = N_sources_selected * N_stations_selected N_times = len(times) N_stations = len(station_names) N_sources = len(source_names) N_pairs = 0 for ii, i in enumerate(source_selection): for jj, j in enumerate(source_selection): if j == i: break subband_selection = find(mask[i,:] * mask[j,:]) if len(subband_selection) < nband_min: continue N_pairs += 1 D = np.resize(station_positions, (N_stations, N_stations, 3)) D = np.transpose(D, (1, 0, 2)) - D D = np.sqrt(np.sum(D**2, axis=2)) station_selection1 = station_selection stations_to_add = np.array([i for i in xrange(len(station_names)) if i not in station_selection1 and station_names[i] not in excluded_stations]) if len(stations_to_add) == 0: return station_selection1, r # Check if desired number of stations is already reached if nstations_max is not None: if len(station_selection1) >= nstations_max: return station_selection1, r logging.info("Using fitting with iterative search for remaining stations " "(up to {0} stations in total)".format(nstations_max)) q = r while len(stations_to_add)>0: D1 = D[stations_to_add[:,newaxis], station_selection1[newaxis,:]] minimum_distance = amin(D1, axis=1) station_to_add = stations_to_add[np.argmin(minimum_distance)] station_selection1 = np.append(station_selection1, station_to_add) N_stations_selected1 = len(station_selection1) # Remove station from list stations_to_add = stations_to_add[stations_to_add != station_to_add] sols_list = [] eq_list = [] min_e_list = [] ipbar = 0 logging.info('Fitting TEC values with {0} included...'.format( station_names[station_to_add])) pbar = progressbar.ProgressBar(maxval=N_pairs*N_times).start() for ii, i in enumerate(source_selection): for jj, j in enumerate(source_selection): if j == i: break subband_selection = find(mask[i,:] * mask[j,:]) if len(subband_selection) < nband_min: continue logging.debug('Adding {0} for source pair: {1}-{2}'.format( station_names[station_to_add], i, j)) p0 = phases0[i, station_selection1[:,newaxis], subband_selection[newaxis,:], :] - phases0[j, station_selection1[:,newaxis], subband_selection[newaxis,:], :] p0 = p0 - np.mean(p0, axis=0)[newaxis,:,:] if soln_type != 'scalarphase': p1 = phases1[i, station_selection1[:,newaxis], subband_selection[newaxis,:], :] - phases1[j, station_selection1[:,newaxis], subband_selection[newaxis,:], :] p1 = p1 - np.mean(p1, axis=0)[newaxis, :, :] A = np.zeros((len(subband_selection), 1)) A[:, 0] = 8.44797245e9 / freqs[subband_selection] sd = (0.1 * 30e6) / freqs[subband_selection] # standard deviation of phase solutions as function of frequency (rad) flags_source_pair = flags[i, station_selection1[:,newaxis], subband_selection[newaxis,:], :] * flags[j, station_selection1[:,newaxis], subband_selection[newaxis,:], :] constant_parms = np.zeros((1, N_stations_selected1), dtype = np.bool) sols = np.zeros((N_times, N_stations_selected1), dtype = np.float) p_0_best = None for t_idx in xrange(N_times): if np.mod(t_idx, t_step) == 0: min_e = np.Inf if p_0_best is not None and not search_full_tec_range: min_tec = p_0_best[0, -1] - 0.02 max_tec = p_0_best[0, -1] + 0.02 nsteps = tec_step1 else: min_tec = -0.1 max_tec = 0.1 nsteps = tec_step2 logging.debug(' Trying initial guesses between {0} and ' '{1} TECU'.format(min_tec, max_tec)) for offset in np.linspace(min_tec, max_tec, nsteps): p_0 = np.zeros((1, N_stations_selected1), np.double) p_0[0, :N_stations_selected1-1] = (q[ii, t_idx, :] - q[jj, t_idx, :])[newaxis,:] p_0[0, -1] = offset x = p0[:,:,t_idx].copy() f = flags_source_pair[:, :, t_idx].copy() sol0 = baselinefitting.fit(x.T, A, p_0, f, constant_parms) sol0 -= np.mean(sol0) residual = np.mod(np.dot(A, sol0) - x.T + np.pi, 2 * np.pi) - np.pi e = np.var(residual[f.T==0]) if soln_type != 'scalarphase': x = p1[:,:,t_idx].copy() f = flags_source_pair[:, :, t_idx].copy() sol1 = baselinefitting.fit(x.T, A, p_0, f, constant_parms) sol1 -= np.mean(sol1) residual = np.mod(np.dot(A, sol1) - x.T + np.pi, 2 * np.pi) - np.pi residual = residual[f.T==0] e += np.var(residual) else: sol1 = sol0 if e < min_e: logging.debug(' Found new min variance of {0} ' 'with initial guess of {1} TECU'.format(e, p_0[0, -1])) min_e = e p_0_best = p_0 sols[t_idx, :] = (sol0[0, :] + sol1[0, :])/2 else: # Use previous init x = p0[:, :, t_idx].copy() f = flags_source_pair[:, :, t_idx].copy() sol0 = baselinefitting.fit(x.T, A, p_0_best, f, constant_parms) sol0 -= np.mean(sol0) if soln_type != 'scalarphase': x = p1[:, :, t_idx].copy() f = flags_source_pair[:, :, t_idx].copy() sol1 = baselinefitting.fit(x.T, A, p_0_best, f, constant_parms) sol1 -= np.mean(sol1) else: sol1 = sol0 sols[t_idx, :] = (sol0[0, :] + sol1[0, :])/2 ipbar += 1 pbar.update(ipbar) ### Remove outliers logging.debug(' Searching for outliers...') for kk in xrange(10): s = sols[:, -1].copy() selection = np.zeros(len(s), np.bool) for t_idx in xrange(len(s)): start_idx = np.max([t_idx-10, 0]) end_idx = np.min([t_idx+10, len(s)]) selection[t_idx] = np.sum(abs(s[start_idx:end_idx] - s[t_idx]) < 0.02) > (end_idx - start_idx - 8) outliers = find(np.logical_not(selection)) if len(outliers) == 0: break for t_idx in outliers: try: idx0 = find(selection[:t_idx])[-1] except IndexError: idx0 = -1 try: idx1 = find(selection[t_idx+1:])[0] + t_idx + 1 except IndexError: idx1 = -1 if idx0 == -1: s[t_idx] = s[idx1] elif idx1 == -1: s[t_idx] = s[idx0] else: s[t_idx] = (s[idx0] * (idx1-t_idx) + s[idx1] * (t_idx-idx0)) / (idx1-idx0) p_0 = np.zeros((1, N_stations_selected1), np.double) p_0[0,:] = sols[t_idx,:] p_0[0,-1] = s[t_idx] x = p0[:,:,t_idx].copy() f = flags_source_pair[:,:,t_idx].copy() sol0 = baselinefitting.fit(x.T, A, p_0, f, constant_parms) sol0 -= np.mean(sol0) if soln_type != 'scalarphase': x = p1[:,:,t_idx].copy() sol1 = baselinefitting.fit(x.T, A, p_0, f, constant_parms) sol1 -= np.mean(sol1) else: sol1 = sol0 sols[t_idx, :] = (sol0[0,:] + sol1[0,:])/2 weight = 1.0 sols_list.append(weight*sols) min_e_list.append(min_e) eq = np.zeros(N_sources) eq[ii] = weight eq[jj] = -weight eq_list.append(eq) sols = np.array(sols_list) B = np.array(eq_list) pinvB = pinv(B) q = np.dot(pinvB, sols.transpose([1,0,2])) pbar.finish() if nstations_max is not None: if N_stations_selected1 == nstations_max: break return station_selection1, q
def fit_tec_per_source_pair(phases, flags, mask, freqs, init_sols=None, init_sols_per_pair=False, propagate=False, nband_min=2): """Fits TEC values to phase solutions per source pair Returns TEC solutions as array of shape (N_sources, N_times, N_stations) Keyword arguments: phases -- array of phase solutions flags -- phase solution flags (0 = use, 1 = flagged) mask -- mask for sources and frequencies (0 = ignore, 1 = use) freqs -- array of frequencies init_sols -- solutions to use to initialize the fits init_sols_per_pair -- init_sols are per source pair (not per source) propagate -- propagate solutions from previous solution nband_min -- min number of bands for a source to be used """ from pylab import pinv, newaxis, find import numpy as np from lofar.expion import baselinefitting try: import progressbar except ImportError: import losoto.progressbar as progressbar sols_list = [] eq_list = [] vars_list = [] var_eq_list = [] N_sources = phases.shape[0] N_stations = phases.shape[1] N_times = phases.shape[3] N_pairs = 0 for i in xrange(N_sources): for j in xrange(i): subband_selection = find(mask[i, :] * mask[j, :]) if len(subband_selection) < nband_min: continue N_pairs += 1 if init_sols is None and not init_sols_per_pair: init_sols = np.zeros((N_sources, N_times, N_stations), dtype = np.float) elif init_sols is None and init_sols_per_pair: init_sols = np.zeros((N_pairs, N_times, N_stations), dtype = np.float) source_pairs = [] k = 0 ipbar = 0 logging.info('Fitting TEC values...') pbar = progressbar.ProgressBar(maxval=N_pairs*N_times).start() for i in xrange(N_sources): for j in xrange(i): subband_selection = find(mask[i, :] * mask[j, :]) if len(subband_selection) < nband_min: continue source_pairs.append((i, j)) p = phases[i, :, subband_selection, :] - phases[j, :, subband_selection, :] A = np.zeros((len(subband_selection), 1)) A[:, 0] = 8.44797245e9/freqs[subband_selection] flags_source_pair = flags[i, :, subband_selection, :] * flags[j, :, subband_selection, :] constant_parms = np.zeros((1, N_stations), dtype = np.bool) sols = np.zeros((N_times, N_stations), dtype = np.float) if init_sols_per_pair: p_0 = init_sols[k, 0, :][newaxis, :] else: p_0 = (init_sols[i, 0, :] - init_sols[j, 0, :])[newaxis, :] for t_idx in xrange(N_times): x = p[:, :, t_idx].copy() f = flags_source_pair[:, :, t_idx].copy() if not propagate: if init_sols_per_pair: p_0 = init_sols[k, t_idx, :][newaxis, :] else: p_0 = (init_sols[i, t_idx, :] - init_sols[j, 0, :])[newaxis, :] sol = baselinefitting.fit(x, A, p_0, f, constant_parms) if propagate: p_0 = sol.copy() sols[t_idx, :] = sol[0, :] ipbar += 1 pbar.update(ipbar) sols = sols[:, :] - np.mean(sols[:, :], axis=1)[:, newaxis] weight = len(subband_selection) sols_list.append(sols*weight) eq = np.zeros(N_sources) eq[i] = weight eq[j] = -weight eq_list.append(eq) k += 1 pbar.finish() sols = np.array(sols_list) B = np.array(eq_list) source_selection = find(np.sum(abs(B), axis=0)) N_sources = len(source_selection) if N_sources == 0: logging.error('All sources have fewer than the required minimum number of bands.') return None, None pinvB = pinv(B) r = np.dot(pinvB, sols.transpose([1, 0, 2])) r = r[source_selection, :, :] return r, source_selection
def add_stations(station_selection, phases0, phases1, flags, mask, station_names, station_positions, source_names, source_selection, times, freqs, r, nband_min=2, soln_type='phase', nstations_max=None, excluded_stations=None, t_step=5, tec_step1=5, tec_step2=21, search_full_tec_range=False): """ Adds stations to TEC fitting using an iterative initial-guess search to ensure the global min is found Keyword arguments: station_selection -- indices of stations to use in fitting phases0 -- XX phase solutions phases1 -- YY phase solutions flags -- phase solution flags (0 = use, 1 = flagged) mask -- mask for sources and frequencies (0 = ignore, 1 = use) station_names -- array of station names source_names -- array of source names source_selection -- indices of sources to use in fitting times -- array of times freqs -- array of frequencies r -- array of TEC solutions returned by fit_tec_per_source_pair() nband_min -- min number of bands for a source to be used soln_type -- type of phase solution: 'phase' or 'scalarphase' nstations_max -- max number of stations to use excluded_stations -- stations to exclude t_step -- try full TEC range every t_step number of solution times tec_step1 -- number of steps in TEC subrange (+/- last TEC fit value) tec_step2 -- number of steps in full TEC range (-0.1 -- 0.1) search_full_tec_range -- always search the full TEC range (-0.1 -- 0.1) """ from pylab import pinv, newaxis, find, amin import numpy as np from lofar.expion import baselinefitting import progressbar N_sources_selected = len(source_selection) N_stations_selected = len(station_selection) N_piercepoints = N_sources_selected * N_stations_selected N_times = len(times) N_stations = len(station_names) N_sources = len(source_names) N_pairs = 0 for ii, i in enumerate(source_selection): for jj, j in enumerate(source_selection): if j == i: break subband_selection = find(mask[i, :] * mask[j, :]) if len(subband_selection) < nband_min: continue N_pairs += 1 D = np.resize(station_positions, (N_stations, N_stations, 3)) D = np.transpose(D, (1, 0, 2)) - D D = np.sqrt(np.sum(D**2, axis=2)) station_selection1 = station_selection stations_to_add = np.array([ i for i in range(len(station_names)) if i not in station_selection1 and station_names[i] not in excluded_stations ]) if len(stations_to_add) == 0: return station_selection1, r # Check if desired number of stations is already reached if nstations_max is not None: if len(station_selection1) >= nstations_max: return station_selection1, r logging.info("Using fitting with iterative search for remaining stations " "(up to {0} stations in total)".format(nstations_max)) q = r while len(stations_to_add) > 0: D1 = D[stations_to_add[:, newaxis], station_selection1[newaxis, :]] minimum_distance = amin(D1, axis=1) station_to_add = stations_to_add[np.argmin(minimum_distance)] station_selection1 = np.append(station_selection1, station_to_add) N_stations_selected1 = len(station_selection1) # Remove station from list stations_to_add = stations_to_add[stations_to_add != station_to_add] sols_list = [] eq_list = [] min_e_list = [] ipbar = 0 logging.info('Fitting TEC values with {0} included...'.format( station_names[station_to_add])) pbar = progressbar.ProgressBar(maxval=N_pairs * N_times).start() for ii, i in enumerate(source_selection): for jj, j in enumerate(source_selection): if j == i: break subband_selection = find(mask[i, :] * mask[j, :]) if len(subband_selection) < nband_min: continue logging.debug('Adding {0} for source pair: {1}-{2}'.format( station_names[station_to_add], i, j)) p0 = phases0[i, station_selection1[:, newaxis], subband_selection[newaxis, :], :] - phases0[ j, station_selection1[:, newaxis], subband_selection[newaxis, :], :] p0 = p0 - np.mean(p0, axis=0)[newaxis, :, :] if soln_type != 'scalarphase': p1 = phases1[i, station_selection1[:, newaxis], subband_selection[newaxis, :], :] - phases1[ j, station_selection1[:, newaxis], subband_selection[newaxis, :], :] p1 = p1 - np.mean(p1, axis=0)[newaxis, :, :] A = np.zeros((len(subband_selection), 1)) A[:, 0] = 8.44797245e9 / freqs[subband_selection] flags_source_pair = flags[ i, station_selection1[:, newaxis], subband_selection[newaxis, :], :] * flags[ j, station_selection1[:, newaxis], subband_selection[newaxis, :], :] constant_parms = np.zeros((1, N_stations_selected1), dtype=np.bool) sols = np.zeros((N_times, N_stations_selected1), dtype=np.float) p_0_best = None for t_idx in range(N_times): if np.mod(t_idx, t_step) == 0: min_e = np.Inf if p_0_best is not None and not search_full_tec_range: min_tec = p_0_best[0, -1] - 0.02 max_tec = p_0_best[0, -1] + 0.02 nsteps = tec_step1 else: min_tec = -0.1 max_tec = 0.1 nsteps = tec_step2 logging.debug( ' Trying initial guesses between {0} and ' '{1} TECU'.format(min_tec, max_tec)) for offset in np.linspace(min_tec, max_tec, nsteps): p_0 = np.zeros((1, N_stations_selected1), np.double) p_0[0, :N_stations_selected1 - 1] = (q[ii, t_idx, :] - q[jj, t_idx, :])[newaxis, :] p_0[0, -1] = offset x = p0[:, :, t_idx].copy() f = flags_source_pair[:, :, t_idx].copy() sol0 = baselinefitting.fit(x.T, A, p_0, f, constant_parms) sol0 -= np.mean(sol0) residual = np.mod( np.dot(A, sol0) - x.T + np.pi, 2 * np.pi) - np.pi residual = residual[f.T == 0] e = np.var(residual) if soln_type != 'scalarphase': x = p1[:, :, t_idx].copy() f = flags_source_pair[:, :, t_idx].copy() sol1 = baselinefitting.fit( x.T, A, p_0, f, constant_parms) sol1 -= np.mean(sol1) residual = np.mod( np.dot(A, sol1) - x.T + np.pi, 2 * np.pi) - np.pi residual = residual[f.T == 0] e += np.var(residual) else: sol1 = sol0 if e < min_e: logging.debug( ' Found new min variance of {0} ' 'with initial guess of {1} TECU'.format( e, p_0[0, -1])) min_e = e p_0_best = p_0 sols[t_idx, :] = (sol0[0, :] + sol1[0, :]) / 2 else: # Use previous init x = p0[:, :, t_idx].copy() f = flags_source_pair[:, :, t_idx].copy() sol0 = baselinefitting.fit(x.T, A, p_0_best, f, constant_parms) sol0 -= np.mean(sol0) if soln_type != 'scalarphase': x = p1[:, :, t_idx].copy() f = flags_source_pair[:, :, t_idx].copy() sol1 = baselinefitting.fit(x.T, A, p_0_best, f, constant_parms) sol1 -= np.mean(sol1) else: sol1 = sol0 sols[t_idx, :] = (sol0[0, :] + sol1[0, :]) / 2 ipbar += 1 pbar.update(ipbar) ### Remove outliers logging.debug(' Searching for outliers...') for kk in range(10): s = sols[:, -1].copy() selection = np.zeros(len(s), np.bool) for t_idx in range(len(s)): start_idx = np.max([t_idx - 10, 0]) end_idx = np.min([t_idx + 10, len(s)]) selection[t_idx] = np.sum( abs(s[start_idx:end_idx] - s[t_idx]) < 0.02) > ( end_idx - start_idx - 8) outliers = find(np.logical_not(selection)) if len(outliers) == 0: break for t_idx in outliers: try: idx0 = find(selection[:t_idx])[-1] except IndexError: idx0 = -1 try: idx1 = find(selection[t_idx + 1:])[0] + t_idx + 1 except IndexError: idx1 = -1 if idx0 == -1: s[t_idx] = s[idx1] elif idx1 == -1: s[t_idx] = s[idx0] else: s[t_idx] = (s[idx0] * (idx1 - t_idx) + s[idx1] * (t_idx - idx0)) / (idx1 - idx0) p_0 = np.zeros((1, N_stations_selected1), np.double) p_0[0, :] = sols[t_idx, :] p_0[0, -1] = s[t_idx] x = p0[:, :, t_idx].copy() f = flags_source_pair[:, :, t_idx].copy() sol0 = baselinefitting.fit(x.T, A, p_0, f, constant_parms) sol0 -= np.mean(sol0) if soln_type != 'scalarphase': x = p1[:, :, t_idx].copy() sol1 = baselinefitting.fit(x.T, A, p_0, f, constant_parms) sol1 -= np.mean(sol1) else: sol1 = sol0 sols[t_idx, :] = (sol0[0, :] + sol1[0, :]) / 2 weight = 1.0 sols_list.append(weight * sols) min_e_list.append(min_e) eq = np.zeros(N_sources) eq[ii] = weight eq[jj] = -weight eq_list.append(eq) sols = np.array(sols_list) B = np.array(eq_list) pinvB = pinv(B) q = np.dot(pinvB, sols.transpose([1, 0, 2])) pbar.finish() if nstations_max is not None: if N_stations_selected1 == nstations_max: break return station_selection1, q
def fit_tec_per_source_pair(phases, flags, mask, freqs, init_sols=None, init_sols_per_pair=False, propagate=False, nband_min=2): """Fits TEC values to phase solutions per source pair Returns TEC solutions as array of shape (N_sources, N_times, N_stations) Keyword arguments: phases -- array of phase solutions flags -- phase solution flags (0 = use, 1 = flagged) mask -- mask for sources and frequencies (0 = ignore, 1 = use) freqs -- array of frequencies init_sols -- solutions to use to initialize the fits init_sols_per_pair -- init_sols are per source pair (not per source) propagate -- propagate solutions from previous solution nband_min -- min number of bands for a source to be used """ from pylab import pinv, newaxis, find import numpy as np from lofar.expion import baselinefitting import progressbar sols_list = [] eq_list = [] vars_list = [] var_eq_list = [] N_sources = phases.shape[0] N_stations = phases.shape[1] N_times = phases.shape[3] N_pairs = 0 for i in range(N_sources): for j in range(i): subband_selection = find(mask[i, :] * mask[j, :]) if len(subband_selection) < nband_min: continue N_pairs += 1 if init_sols is None and not init_sols_per_pair: init_sols = np.zeros((N_sources, N_times, N_stations), dtype=np.float) elif init_sols is None and init_sols_per_pair: init_sols = np.zeros((N_pairs, N_times, N_stations), dtype=np.float) source_pairs = [] k = 0 ipbar = 0 logging.info('Fitting TEC values...') pbar = progressbar.ProgressBar(maxval=N_pairs * N_times).start() for i in range(N_sources): for j in range(i): subband_selection = find(mask[i, :] * mask[j, :]) if len(subband_selection) < nband_min: continue source_pairs.append((i, j)) p = phases[i, :, subband_selection, :] - phases[j, :, subband_selection, :] A = np.zeros((len(subband_selection), 1)) A[:, 0] = 8.44797245e9 / freqs[subband_selection] flags_source_pair = flags[i, :, subband_selection, :] * flags[ j, :, subband_selection, :] constant_parms = np.zeros((1, N_stations), dtype=np.bool) sols = np.zeros((N_times, N_stations), dtype=np.float) if init_sols_per_pair: p_0 = init_sols[k, 0, :][newaxis, :] else: p_0 = (init_sols[i, 0, :] - init_sols[j, 0, :])[newaxis, :] for t_idx in range(N_times): x = p[:, :, t_idx].copy() f = flags_source_pair[:, :, t_idx].copy() if not propagate: if init_sols_per_pair: p_0 = init_sols[k, t_idx, :][newaxis, :] else: p_0 = (init_sols[i, t_idx, :] - init_sols[j, 0, :])[newaxis, :] sol = baselinefitting.fit(x, A, p_0, f, constant_parms) if propagate: p_0 = sol.copy() sols[t_idx, :] = sol[0, :] ipbar += 1 pbar.update(ipbar) sols = sols[:, :] - np.mean(sols[:, :], axis=1)[:, newaxis] weight = len(subband_selection) sols_list.append(sols * weight) eq = np.zeros(N_sources) eq[i] = weight eq[j] = -weight eq_list.append(eq) k += 1 pbar.finish() sols = np.array(sols_list) B = np.array(eq_list) source_selection = find(np.sum(abs(B), axis=0)) N_sources = len(source_selection) if N_sources == 0: logging.error( 'All sources have fewer than the required minimum number of bands.' ) return None, None pinvB = pinv(B) r = np.dot(pinvB, sols.transpose([1, 0, 2])) r = r[source_selection, :, :] return r, source_selection
def getClockTECBaselineFit(ph,amp,freqs,SBselect,polIdx,stIdx,useOffset=False,stations=[],initSol=[],chi2cut=300.,fixedClockforCS=False,timeIdx=0): global tecarray global clockarray global offsetarray global residualarray amp[np.isnan(ph)]=1 ph[np.isnan(ph)]=0. ph=np.unwrap(ph,axis=0) #first unwrap data and get initial guess nT=ph.shape[0] nF=freqs.shape[0] nSt=ph.shape[2] nparms=2+(useOffset>0) #sol = np.zeros((nSt,nparms),dtype=np.float) sol = np.zeros((nSt,nparms),dtype=np.float) print sol.shape,nparms,nSt A=np.zeros((nF,nparms),dtype=np.float) A[:,1] = freqs*2*np.pi*(-1e-9) A[:,0] = -8.44797245e9/freqs if useOffset: A[:,2] = np.ones((nF,)) constant_parms=np.zeros(sol.shape,dtype=bool) if fixedClockforCS: for ist,st in enumerate(stations): if 'CS' in st: constant_parms[ist,1]=True stepDelay=1 if "HBA" in stations[0]: stepdTEC=0.005 else: stepdTEC=0.001 # faster wrapping for LBA stepDelay=3 succes=False initprevsol=False nrFail=0 for itm in range(nT): if itm%100==0 and itm>0: sys.stdout.write(str(itm)+'... '+str(sol[-1,0]-sol[0,0])+' '+str(sol[-1,1]-sol[0,1])+' '+str(sol[-1,-1]-sol[0,-1])+' ') sys.stdout.flush() flags=(amp[itm,:]==1); nrFlags=np.sum(flags,axis=0) if itm==0 or not succes: for ist in range(1,nSt): if (nF-nrFlags[ist])<10: print "Too many data points flagged",itm,ist continue; if itm==0 or not initprevsol: if hasattr(initSol,'__len__') and len(initSol)>ist: iTEC1=initSol[ist,0] iTEC2=initSol[ist,0]+stepdTEC iD1=initSol[ist,1] iD2=initSol[ist,1]+stepDelay else: if 'CS' in stations[ist]: iTEC1=-0.2 iTEC2=0.2 iD1=-4 iD2=4 else: iTEC1=-1.5 iTEC2=1.5 iD1=-50 iD2=300 print "First",iTEC1,iTEC2,iD1,iD2 else: iTEC1=prevsol[ist,0]-stepdTEC*nrFail iTEC2=prevsol[ist,0]+stepdTEC*(nrFail+1) if not fixedClockforCS or not 'CS' in stations[ist]: iD1=prevsol[ist,1]-stepDelay*nrFail iD2=prevsol[ist,1]+stepDelay*(nrFail+1) else: iD1=sol[ist,1] iD2=sol[ist,1]+stepDelay print "Failure",iTEC1,iTEC2,iD1,iD2,nrFail dTECArray=np.arange(iTEC1,iTEC2,stepdTEC) dClockArray=np.arange(iD1,iD2,stepDelay) data=ph[itm,:,ist][np.logical_not(np.logical_or(flags[:,ist],flags[:,0]))]-ph[itm,:,0][np.logical_not(np.logical_or(flags[:,ist],flags[:,0]))] tmpfreqs=freqs[np.logical_not(np.logical_or(flags[:,ist],flags[:,0]))] print "getting init",ist, par = getInitPar(data,dTECArray, dClockArray,tmpfreqs,ClockTECfunc) print par sol[ist,:]=par[:nparms] if not succes: #reset first station sol[0,:] = np.zeros(nparms) #sol=fitting.fit(ph[itm],A,sol.T,flags).T #sol=fitting.fit(ph[itm],A,sol,flags) sol=fitting.fit(ph[itm],A,sol.T,flags,constant_parms.T).T tecarray[itm+timeIdx,stIdx,polIdx]=sol[:,0] clockarray[itm+timeIdx,stIdx,polIdx]=sol[:,1] if useOffset: offsetarray[itm+timeIdx,stIdx,polIdx]+=sol[:,2] residual = ph[itm] - np.dot(A, sol.T) residual = residual - residual[:, 0][:,np.newaxis] residual = np.remainder(residual+np.pi, 2*np.pi) - np.pi residual[flags]=0 residualarray[np.ix_([itm+timeIdx],SBselect,stIdx,[polIdx])]=residual.reshape((1,nF,nSt,1)) chi2=np.sum(np.square(np.degrees(residual)))/(nSt*nF) if chi2>chi2cut: print "failure",chi2,sol succes=False nrFail=0 else: prevsol=np.copy(sol) # print "succes",chi2,dTECArray.shape,dClockArray.shape succes=True initprevsol=True nrFail+=1
def add_stations(station_selection, phases0, phases1, flags, mask, station_names, station_positions, source_names, source_selection, times, freqs, r, nband_min=2, soln_type='phase', nstations_max=None): N_sources_selected = len(source_selection) N_stations_selected = len(station_selection) N_piercepoints = N_sources_selected * N_stations_selected N_times = len(times) N_stations = len(station_names) N_sources = len(source_names) D = resize( station_positions, ( N_stations, N_stations, 3 ) ) D = transpose( D, ( 1, 0, 2 ) ) - D D = sqrt(sum( D**2, axis=2 )) station_selection1 = station_selection stations_to_add = array([i for i in range(len(station_names)) if i not in station_selection1]) # Check if desired number of stations is already reached if nstations_max is not None: if len(station_selection1) >= nstations_max: return station_selection1, None, r q = r while len(stations_to_add)>0: D1 = D[stations_to_add[:,newaxis], station_selection1[newaxis,:]] minimum_distance = amin(D1, axis=1) station_to_add = stations_to_add[argmin(minimum_distance)] station_selection1 = append(station_selection1, station_to_add) N_stations_selected1 = len(station_selection1) # Remove station from list stations_to_add = stations_to_add[stations_to_add != station_to_add] sols_list = [] eq_list = [] min_e_list = [] for ii, i in enumerate(source_selection): for jj, j in enumerate(source_selection): if j == i: break subband_selection = find(mask[i,:] * mask[j,:]) if len(subband_selection) < nband_min: continue logging.info('Fitting {0} TEC values for source pair: {1}-{2}'.format(station_names[station_to_add], i, j)) p0 = phases0[i, station_selection1[:,newaxis], subband_selection[newaxis,:], :] - phases0[j, station_selection1[:,newaxis], subband_selection[newaxis,:], :] p0 = p0 - mean(p0, axis=0)[newaxis,:,:] if soln_type != 'scalarphase': p1 = phases1[i, station_selection1[:,newaxis], subband_selection[newaxis,:], :] - phases1[j, station_selection1[:,newaxis], subband_selection[newaxis,:], :] p1 = p1 - mean(p1, axis=0)[newaxis,:,:] A = zeros((len(subband_selection), 1)) A[:,0] = 8.44797245e9/freqs[subband_selection] flags_source_pair = flags[i, station_selection1[:,newaxis], subband_selection[newaxis,:], :] * flags[j, station_selection1[:,newaxis], subband_selection[newaxis,:], :] constant_parms = zeros((1, N_stations_selected1), dtype = bool) sols = zeros((N_times, N_stations_selected1), dtype = float) for t_idx in range(N_times): if np.mod(t_idx, 10) == 0: min_e = Inf for offset in linspace(-0.1, 0.1,21) : p_0 = zeros((1, N_stations_selected1), double) p_0[0,:N_stations_selected1-1] = (q[ii, t_idx, :] - q[jj, t_idx, :])[newaxis,:] p_0[0,-1] = offset x = p0[:,:,t_idx].copy() f = flags_source_pair[:,:,t_idx].copy() sol0 = baselinefitting.fit(x.T, A, p_0, f, constant_parms) sol0 -= mean(sol0) residual = mod(dot(A,sol0) - x.T + pi, 2*pi) - pi residual = residual[f.T==0] e = var(residual) if soln_type != 'scalarphase': x = p1[:,:,t_idx].copy() f = flags_source_pair[:,:,t_idx].copy() sol1 = baselinefitting.fit(x.T, A, p_0, f, constant_parms) sol1 -= mean(sol1) residual = mod(dot(A,sol1) - x.T + pi, 2*pi) - pi residual = residual[f.T==0] e += var(residual) else: sol1 = sol0 if e<min_e: min_e = e p_0_best = p_0 sols[t_idx, :] = (sol0[0,:] + sol1[0,:])/2 else: # Use previous init x = p0[:,:,t_idx].copy() f = flags_source_pair[:,:,t_idx].copy() sol0 = baselinefitting.fit(x.T, A, p_0_best, f, constant_parms) sol0 -= mean(sol0) if soln_type != 'scalarphase': x = p1[:,:,t_idx].copy() f = flags_source_pair[:,:,t_idx].copy() sol1 = baselinefitting.fit(x.T, A, p_0_best, f, constant_parms) sol1 -= mean(sol1) else: sol1 = sol0 sols[t_idx, :] = (sol0[0,:] + sol1[0,:])/2 ### Remove outliers for kk in range(10): s = sols[:,-1].copy() selection = zeros(len(s), bool) for t_idx in range(len(s)): start_idx = max(t_idx-10, 0) end_idx = min(t_idx+10, len(s)) selection[t_idx] = sum(abs(s[start_idx:end_idx] - s[t_idx])<0.02) > (end_idx - start_idx - 8) outliers = find(logical_not(selection)) if len(outliers) == 0: break for t_idx in outliers: try: idx0 = find(selection[:t_idx])[-1] except IndexError: idx0 = -1 try: idx1 = find(selection[t_idx+1:])[0]+t_idx+1 except IndexError: idx1 = -1 if idx0 == -1: s[t_idx] = s[idx1] elif idx1 == -1: s[t_idx] = s[idx0] else: s[t_idx] = (s[idx0] * (idx1-t_idx) + s[idx1] * (t_idx-idx0)) / (idx1-idx0) p_0 = zeros((1, N_stations_selected1), double) p_0[0,:] = sols[t_idx,:] p_0[0,-1] = s[t_idx] x = p0[:,:,t_idx].copy() f = flags_source_pair[:,:,t_idx].copy() sol0 = baselinefitting.fit(x.T, A, p_0, f, constant_parms) sol0 -= mean(sol0) if soln_type != 'scalarphase': x = p1[:,:,t_idx].copy() sol1 = baselinefitting.fit(x.T, A, p_0, f, constant_parms) sol1 -= mean(sol1) else: sol1 = sol0 sols[t_idx, :] = (sol0[0,:] + sol1[0,:])/2 weight = 1.0 sols_list.append(weight*sols) min_e_list.append(min_e) eq = zeros(N_sources) eq[ii] = weight eq[jj] = -weight eq_list.append(eq) sols = array(sols_list) B = array(eq_list) pinvB = pinv(B) q = dot(pinvB, sols.transpose([1,0,2])) if nstations_max is not None: if N_stations_selected1 == nstations_max: break return station_selection1, sols, q
def fit_tec_per_source_pair(phases, flags, mask, freqs, init_sols=None, init_sols_per_pair=False, propagate=False, nband_min=2, offset=0): sols_list = [] eq_list = [] vars_list = [] var_eq_list = [] N_sources = phases.shape[0] N_stations = phases.shape[1] N_times = phases.shape[3] k = 0 for i in range(N_sources): for j in range(i): k += 1 N_pairs = k if init_sols is None and not init_sols_per_pair: init_sols = zeros((N_sources, N_times, N_stations), dtype = float) + offset elif init_sols is None and init_sols_per_pair: init_sols = zeros((N_pairs, N_times, N_stations), dtype = float) + offset vars = zeros((N_times, N_stations), dtype = float) source_pairs = [] k = 0 for i in range(N_sources): for j in range(i): subband_selection = find(mask[i,:] * mask[j,:]) if len(subband_selection) < nband_min: continue logging.info('Fitting TEC values for source pair: {0}-{1}'.format(i, j)) source_pairs.append((i,j)) p = phases[i, :, subband_selection, :] - phases[j, :, subband_selection, :] A = zeros((len(subband_selection), 1)) A[:,0] = 8.44797245e9/freqs[subband_selection] flags_source_pair = flags[i, :, subband_selection, :] * flags[j, :, subband_selection, :] constant_parms = zeros((1, N_stations), dtype = bool) sols = zeros((N_times, N_stations), dtype = float) if init_sols_per_pair: p_0 = init_sols[k, 0, :][newaxis,:] else: p_0 = (init_sols[i, 0, :] - init_sols[j, 0, :])[newaxis,:] for t_idx in range(N_times): x = p[:,:,t_idx].copy() f = flags_source_pair[:,:,t_idx].copy() if not propagate: if init_sols_per_pair: p_0 = init_sols[k, t_idx, :][newaxis,:] else: p_0 = (init_sols[i, t_idx, :] - init_sols[j, 0, :])[newaxis,:] sol = baselinefitting.fit(x, A, p_0, f, constant_parms) if propagate: p_0 = sol.copy() sols[t_idx, :] = sol[0, :] sols = sols[:, :]-mean(sols[:, :], axis=1)[:,newaxis] weight = len(subband_selection) sols_list.append(sols*weight) vars_list.append(vars*weight) eq = zeros(N_sources) eq[i] = weight eq[j] = -weight eq_list.append(eq) var_eq = zeros(N_sources) var_eq[i] = weight var_eq[j] = weight var_eq_list.append(var_eq) k += 1 sols = array(sols_list) B = array(eq_list) source_selection = find(sum(abs(B), axis=0)) N_sources = len(source_selection) if N_sources == 0: logging.error('No sources with enough bands.') return None, None, None, None pinvB = pinv(B) r = dot(pinvB, sols.transpose([1,0,2])) r = r[source_selection, :, :] return r, source_selection, sols, source_pairs
def getClockTECBaselineFit(ph, amp, freqs, SBselect, polIdx, stIdx, useOffset=False, stations=[], initSol=[], chi2cut=300., fixedClockforCS=False, timeIdx=0): global tecarray global clockarray global offsetarray global residualarray amp[np.isnan(ph)] = 1 ph[np.isnan(ph)] = 0. ph = np.unwrap(ph, axis=0) #first unwrap data and get initial guess nT = ph.shape[0] nF = freqs.shape[0] nSt = ph.shape[2] nparms = 2 + (useOffset > 0) #sol = np.zeros((nSt,nparms),dtype=np.float) sol = np.zeros((nSt, nparms), dtype=np.float) print(sol.shape, nparms, nSt) A = np.zeros((nF, nparms), dtype=np.float) A[:, 1] = freqs * 2 * np.pi * (-1e-9) A[:, 0] = -8.44797245e9 / freqs if useOffset: A[:, 2] = np.ones((nF, )) constant_parms = np.zeros(sol.shape, dtype=bool) if fixedClockforCS: for ist, st in enumerate(stations): if 'CS' in st: constant_parms[ist, 1] = True stepDelay = 1 if "HBA" in stations[0]: stepdTEC = 0.005 else: stepdTEC = 0.001 # faster wrapping for LBA stepDelay = 3 succes = False initprevsol = False nrFail = 0 for itm in range(nT): if itm % 100 == 0 and itm > 0: sys.stdout.write( str(itm) + '... ' + str(sol[-1, 0] - sol[0, 0]) + ' ' + str(sol[-1, 1] - sol[0, 1]) + ' ' + str(sol[-1, -1] - sol[0, -1]) + ' ') sys.stdout.flush() flags = (amp[itm, :] == 1) nrFlags = np.sum(flags, axis=0) if itm == 0 or not succes: for ist in range(1, nSt): if (nF - nrFlags[ist]) < 10: print("Too many data points flagged", itm, ist) continue if itm == 0 or not initprevsol: if hasattr(initSol, '__len__') and len(initSol) > ist: iTEC1 = initSol[ist, 0] iTEC2 = initSol[ist, 0] + stepdTEC iD1 = initSol[ist, 1] iD2 = initSol[ist, 1] + stepDelay else: if 'CS' in stations[ist]: iTEC1 = -0.2 iTEC2 = 0.2 iD1 = -4 iD2 = 4 else: iTEC1 = -1.5 iTEC2 = 1.5 iD1 = -50 iD2 = 300 print("First", iTEC1, iTEC2, iD1, iD2) else: iTEC1 = prevsol[ist, 0] - stepdTEC * nrFail iTEC2 = prevsol[ist, 0] + stepdTEC * (nrFail + 1) if not fixedClockforCS or not 'CS' in stations[ist]: iD1 = prevsol[ist, 1] - stepDelay * nrFail iD2 = prevsol[ist, 1] + stepDelay * (nrFail + 1) else: iD1 = sol[ist, 1] iD2 = sol[ist, 1] + stepDelay print("Failure", iTEC1, iTEC2, iD1, iD2, nrFail) dTECArray = np.arange(iTEC1, iTEC2, stepdTEC) dClockArray = np.arange(iD1, iD2, stepDelay) data = ph[itm, :, ist][np.logical_not( np.logical_or( flags[:, ist], flags[:, 0]))] - ph[itm, :, 0][np.logical_not( np.logical_or(flags[:, ist], flags[:, 0]))] tmpfreqs = freqs[np.logical_not( np.logical_or(flags[:, ist], flags[:, 0]))] print("getting init", ist, end=' ') par = getInitPar(data, dTECArray, dClockArray, tmpfreqs, ClockTECfunc) print(par) sol[ist, :] = par[:nparms] if not succes: #reset first station sol[0, :] = np.zeros(nparms) #sol=fitting.fit(ph[itm],A,sol.T,flags).T #sol=fitting.fit(ph[itm],A,sol,flags) sol = fitting.fit(ph[itm], A, sol.T, flags, constant_parms.T).T tecarray[itm + timeIdx, stIdx, polIdx] = sol[:, 0] clockarray[itm + timeIdx, stIdx, polIdx] = sol[:, 1] if useOffset: offsetarray[itm + timeIdx, stIdx, polIdx] += sol[:, 2] residual = ph[itm] - np.dot(A, sol.T) residual = residual - residual[:, 0][:, np.newaxis] residual = np.remainder(residual + np.pi, 2 * np.pi) - np.pi residual[flags] = 0 residualarray[np.ix_([itm + timeIdx], SBselect, stIdx, [polIdx])] = residual.reshape((1, nF, nSt, 1)) chi2 = np.sum(np.square(np.degrees(residual))) / (nSt * nF) if chi2 > chi2cut: print("failure", chi2, sol) succes = False nrFail = 0 else: prevsol = np.copy(sol) # print "succes",chi2,dTECArray.shape,dClockArray.shape succes = True initprevsol = True nrFail += 1