def _flag(weights,
coord,
axesToExt,
selection,
percent=50,
size=[0],
maxCycles=3,
outQueue=None):
"""
Flag data if surreounded by other flagged data
weights = the weights to convert into flags
percent = percent of surrounding flagged point to extend the flag
return: flags array and final rms
"""
def extendFlag(flags, percent):
#flags = flags.astype(np.int)
if float(np.sum(flags)) / len(flags) > percent / 100.:
return 1
else:
return 0
def percentFlagged(w):
return 100. * (weights.size - np.count_nonzero(weights)) / float(
weights.size)
import scipy.ndimage
import numpy as np
initPercent = percentFlagged(weights)
# if size=0 then extend to all 2*axis, this otherwise create issues with mirroring
for i, s in enumerate(size):
if s == 0: size[i] = 2 * weights.shape[i]
for cycle in range(maxCycles):
flag = scipy.ndimage.filters.generic_filter(
(weights == 0),
extendFlag,
size=size,
mode='mirror',
cval=0.0,
origin=0,
extra_keywords={'percent': percent})
weights[(flag == 1)] = 0
# no new flags
if cycle != 0 and np.count_nonzero(flag) == oldFlagCount: break
oldFlagCount = np.count_nonzero(flag)
if percentFlagged(weights) == initPercent:
logging.debug('Percentage of data flagged (%s): %.3f -> None' \
% (removeKeys(coord, axesToExt), initPercent))
else:
logging.debug('Percentage of data flagged (%s): %.3f -> %.3f %%' \
% (removeKeys(coord, axesToExt), initPercent, percentFlagged(weights)))
outQueue.put([weights, selection])
def correctWrapsFromResiduals(residualarray,flags,freq):
'''corrects solution jumps due to 2 pi phasewraps based on the average residuals'''
resflags = np.logical_or(flags[:, np.newaxis], residualarray == 0)
maskedresiduals = np.ma.array(residualarray, mask=resflags)
# avgResiduals=np.average(residualarray,axis=0)
avgResiduals = np.ma.average(maskedresiduals, axis=0)
(wraps, steps) = getResidualPhaseWraps(avgResiduals, freq) # fitting of the wraps from the time-avg residuals
# step[0] is the step in TEC corresponding to 1 phase wrap and step[1] is in ns (clock)
wraps = np.round(wraps - wraps[0]) # reference to station 0
logging.debug('Wraps from residuals: ' + str(wraps))
return (wraps, steps)
def run(soltab, axesToNorm, normVal=1., log=False):
"""
Normalize the solutions to a given value
WEIGHT: Weights compliant
Parameters
----------
axesToNorm : array of str
Axes along which compute the normalization.
normVal : float, optional
Number to normalize to vals = vals * (normVal/valsMean), by default 1.
log : bool, optional
clip is done in log10 space, by default False.
"""
import numpy as np
logging.info("Normalizing soltab: " + soltab.name)
# input check
axesNames = soltab.getAxesNames()
for normAxis in axesToNorm:
if normAxis not in axesNames:
logging.error('Normalization axis ' + normAxis + ' not found.')
return 1
if soltab.getType() == 'amplitude' and not log:
logging.warning(
'Amplitude solution tab detected and log=False. Amplitude solution tables should be treated in log space.'
)
for vals, weights, coord, selection in soltab.getValuesIter(
returnAxes=axesToNorm, weight=True):
if log: vals = np.log10(vals)
# rescale solutions
if np.all(weights == 0): continue # skip flagged selections
valsMean = np.nanmean(vals[weights != 0])
if log: vals[weights != 0] += np.log10(normVal) - valsMean
else: vals[weights != 0] *= normVal / valsMean
logging.debug("Rescaling by: " + str(normVal / valsMean))
# writing back the solutions
if log: vals = 10**vals
soltab.setValues(vals, selection)
soltab.flush()
soltab.addHistory('NORM (on axis %s)' % (axesToNorm))
return 0
def run( soltab, ms, inverse=False, useElementResponse=True, useArrayFactor=True, useChanFreq=True ):
"""
Generic unspecified step for easy expansion.
Parameters
----------
opt1 : float
Is a mandatory parameter.
"""
# load specific libs
import numpy as np
import casacore.tables as pt
from lofar.stationresponse import stationresponse
sr = stationresponse(ms, inverse, useElementResponse, useArrayFactor, useChanFreq)
numants = pt.taql('select gcount(*) as numants from '+ms+'::ANTENNA').getcol('numants')[0]
times = soltab.getAxisValues('time')
for vals, coord, selection in soltab.getValuesIter(returnAxes=['ant','time','pol','freq'], weight=False):
vals = reorderAxes( vals, soltab.getAxesNames(), ['ant','time','freq','pol'])
for stationnum in range(numants):
logging.debug('Working on station number %i' % stationnum)
for itime, time in enumerate(times):
beam = sr.evaluateStation(time=time, station=stationnum)
# Reshape from [nfreq, 2, 2] to [nfreq, 4]
beam = beam.reshape(beam.shape[0], 4)
if soltab.getAxisLen('pol') == 2:
beam = beam[:,[0,3]] # get only XX and YY
if soltab.getType() == 'amplitude':
vals[stationnum, itime, :, :] = np.abs(beam)
elif soltab.getType() == 'phase':
vals[stationnum, itime, :, :] = np.angle(beam)
else:
logging.error('Beam prediction work only for amplitude/phase solution tables.')
return 1
vals = reorderAxes( vals, ['ant','time','freq','pol'], [ax for ax in soltab.getAxesNames() if ax in ['ant','time','freq','pol']] )
soltab.setValues(vals, selection)
return 0
def getResidualPhaseWraps(avgResiduals, freqs):
flags = np.average(avgResiduals.mask,axis=1)>0.5
nSt = avgResiduals.shape[1]
nF = freqs.shape[0]
wraps = np.zeros((nSt, ), dtype=np.float)
tmpflags = flags
tmpfreqs = freqs[np.logical_not(tmpflags)]
steps=[0,0]
if np.ma.count(tmpfreqs) < 3:
logging.debug('Cannot unwrap, too many channels flagged')
return (wraps,steps)
(tmpbasef, steps) = getPhaseWrapBase(tmpfreqs)
basef = np.zeros(freqs.shape)
basef[np.logical_not(tmpflags)] = tmpbasef
basef = basef.reshape((-1, 1))
data = avgResiduals[:, :]
if has_fitting:
wraps = fitting.fit(data, basef, wraps, flags).flatten()
return (wraps, steps)
def __init__(self, procs=0, funct=None):
"""
Manager for multiprocessing
procs: number of processors, if 0 use all available
funct: function to parallelize / note that the last parameter of this function must be the outQueue
and it will be linked to the output queue
"""
if procs == 0:
procs = multiprocessing.cpu_count()
self.procs = procs
self._threads = []
self.inQueue = multiprocessing.JoinableQueue()
self.outQueue = multiprocessing.Queue()
self.runs = 0
logging.debug('Spawning %i threads...' % self.procs)
for proc in range(self.procs):
t = self.multiThread(self.inQueue, self.outQueue, funct)
self._threads.append(t)
t.start()
def run(soltab, axesToNorm, normVal=1.):
"""
Normalize the solutions to a given value
WEIGHT: Weights compliant
Parameters
----------
axesToNorm : array of str
Axes along which compute the normalization.
normVal : float, optional
Number to normalize to vals = vals * (normVal/valsMean), by default 1.
"""
import numpy as np
logging.info("Normalizing soltab: " + soltab.name)
# input check
axesNames = soltab.getAxesNames()
for normAxis in axesToNorm:
if normAxis not in axesNames:
logging.error('Normalization axis ' + normAxis + ' not found.')
return 1
for vals, weights, coord, selection in soltab.getValuesIter(
returnAxes=axesToNorm, weight=True):
# rescale solutions
if np.all(weights == 0): continue # skip flagged selections
valsMean = np.nanmean(vals[weights != 0])
vals[weights != 0] *= normVal / valsMean
logging.debug(str(coord))
logging.debug("Rescaling by: " + str(normVal / valsMean))
# writing back the solutions
soltab.setValues(vals, selection)
soltab.flush()
soltab.addHistory('NORM (on axis %s)' % (axesToNorm))
return 0
def _run_timestep(t,coord_rr,coord_ll,weights,vals,solType,coord,maxResidual):
c = 2.99792458e8
if solType == 'phase':
idx = ((weights[coord_rr,:] != 0.) & (weights[coord_ll,:] != 0.))
freq = np.copy(coord['freq'])[idx]
phase_rr = vals[coord_rr,:][idx]
phase_ll = vals[coord_ll,:][idx]
# RR-LL to be consistent with BBS/NDPPP
phase_diff = (phase_rr - phase_ll) # not divide by 2 otherwise jump problem, then later fix this
else: # rotation table
idx = ((weights[:] != 0.) & (weights[:] != 0.))
freq = np.copy(coord['freq'])[idx]
phase_diff = 2.*vals[:][idx] # a rotation is between -pi and +pi
if len(freq) < 20:
fitresultrm_wav = [0]
weight = 0
logging.warning('No valid data found for Faraday fitting for antenna: '+coord['ant']+' at timestamp '+str(t))
else:
# if more than 1/4 of chans are flagged
if (len(idx) - len(freq))/float(len(idx)) > 1/4.:
logging.debug('High number of filtered out data points for the timeslot %i: %i/%i' % (t, len(idx) - len(freq), len(idx)) )
wav = c/freq
#fitresultrm_wav, success = scipy.optimize.leastsq(rmwavcomplex, [fitrmguess], args=(wav, phase_diff))
ranges = slice(-0.1, 0.1, 2e-4)
fitresultrm_wav = scipy.optimize.brute(costfunctionRM, (ranges,), finish=scipy.optimize.leastsq, args=(wav, phase_diff))
# fractional residual
residual = np.nanmean(np.abs(np.mod((2.*fitresultrm_wav*wav*wav)-phase_diff + np.pi, 2.*np.pi) - np.pi))
if maxResidual == 0 or residual < maxResidual:
fitrmguess = fitresultrm_wav[0] # Don't know what this is for...
weight = 1
else:
# high residual, flag
logging.warning('Bad solution for ant: '+coord['ant']+' (time: '+str(t)+', resdiaul: '+str(residual)+').')
weight = 0
return fitresultrm_wav[0],weight
def correctWraps(
tecarray,
residualarray,
freq,
pos,
):
'''corrects solution jumps due to 2 pi phasewraps based on spatial correlations of averaged TEC solutions. Also returns average constant phase offset per station. International stations should be excluded from this'''
nT = tecarray.shape[0]
nSt = tecarray.shape[1]
flags = tecarray < -5
(wraps, steps) = correctWrapsFromResiduals(residualarray, flags, freq)
lats = np.degrees(
np.arctan2(pos[:, 2],
np.sqrt(pos[:, 0] * pos[:, 0] + pos[:, 1] * pos[:, 1])))
lats -= lats[0]
lons = np.degrees(np.arctan2(pos[:, 1], pos[:, 0]))
lons -= lons[0]
lonlat = np.concatenate((lons, lats)).reshape((2, ) + lons.shape)
# refine (is it needed/helpful for LBA? the offsets seem to make sense for LBA)
myselect = np.sqrt(np.sum(lonlat**2, axis=0)) > 0.5
for nr_iter in range(2):
# recreate best TEC at the moment
TEC = tecarray - tecarray[:, [0]] + steps[0] * (np.round(wraps) -
np.round(wraps[0]))
#TEC = np.ma.array(TEC, mask=flags)
TEC = np.ma.array(TEC, mask=np.logical_or(flags, myselect[np.newaxis]))
# fit 2d linear TEC screen over stations
slope = np.ma.dot(np.linalg.inv(np.dot(lonlat, lonlat.T)),
np.ma.dot(lonlat, TEC.T))
# flag bad time steps maybe because TEC screen is not linear
chi2 = np.ma.sum(np.square(TEC - np.ma.dot(lonlat.T, slope).T),
axis=1) / nSt
chi2select = chi2 < np.ma.average(chi2)
chi2select = chi2 < np.ma.average(chi2[chi2select])
# calculate offset per station wrt time-averaged TEC screen
offsets = -1 * np.ma.average(
TEC[chi2select] - np.ma.dot(slope.T, lonlat)[chi2select],
axis=0) * 2. * np.pi / steps[0]
remainingwraps = np.round(
offsets / (2 * np.pi)) # -np.round(wraps[stationIndices])
offsets[myselect] = 0
remainingwraps[myselect] = 0
logging.debug('Offsets: ' + str(offsets))
logging.debug('AvgTEC: ' + str(np.ma.average(TEC[chi2select], axis=0)))
logging.debug('Remaining: ' + str(remainingwraps))
wraps += remainingwraps
# TODO: remove also the offset before the second cycle
if np.sum(np.absolute(remainingwraps)) == 0:
break
return (offsets, wraps, steps)
#!/usr/bin/env python
# -*- coding: utf-8 -*-
from losoto.lib_operations import *
from losoto._logging import logger as logging
logging.debug('Loading DELETEAXIS module.')
def _run_parser(soltab, parser, step):
axisDelete = parser.getstr( step, 'axisDelete' ) # no default
fromCell = parser.getstr( step, 'fromCell', '' )
parser.checkSpelling( step, soltab, ['axisDelete', 'fromCell'])
return run(soltab, axisDelete, fromCell)
def run( soltab, axisDelete, fromCell):
"""
Delete an axis only keeping the values of a certain slice.
Parameters
----------
axisDelete : str
Axis to delete.
fromCell : str
A cell value in axisDelete from which to keep the data values. If it is the string
"first"/"last" then uses the first/last element of the axis.
"""
import numpy as np
if not axisDelete in soltab.getAxesNames():
def run(soltab, doUnwrap=False, refAnt='', plotName='', ndiv=1):
"""
Find the structure function from phase solutions of core stations.
Parameters
----------
doUnwrap : bool, optional
refAnt : str, optional
Reference antenna, by default the first.
plotName : str, optional
Plot file name, by default no plot.
ndiv : int, optional
"""
import numpy as np
from losoto.lib_unwrap import unwrap, unwrap_2d
logging.info("Find structure function for soltab: " + soltab.name)
# input check
solType = soltab.getType()
if solType != 'phase':
logging.warning("Soltab type of " + soltab._v_name + " is of type " +
solType + ", should be phase.")
return 1
ants = soltab.getAxisValues('ant')
if refAnt != '' and refAnt != 'closest' and not refAnt in soltab.getAxisValues(
'ant', ignoreSelection=True):
logging.error('Reference antenna ' + refAnt + ' not found. Using: ' +
ants[1])
refAnt = ants[1]
if refAnt == '' and doUnwrap:
logging.error('Unwrap requires reference antenna. Using: ' + ants[1])
refAnt = ants[1]
if refAnt == '': refAnt = None
soltab.setSelection(ant='CS*', update=True)
posAll = soltab.getSolset().getAnt()
for vals, weights, coord, selection in soltab.getValuesIter(
returnAxes=['freq', 'pol', 'ant', 'time'],
weight=True,
reference=refAnt):
# reorder axes
vals = reorderAxes(vals, soltab.getAxesNames(),
['pol', 'ant', 'freq', 'time'])
weights = reorderAxes(weights, soltab.getAxesNames(),
['pol', 'ant', 'freq', 'time'])
# order positions
pos = np.array([list(posAll[ant]) for ant in coord['ant']])
# avg pols
vals = np.cos(vals) + 1.j * np.sin(vals)
vals = np.nansum(vals, axis=0)
vals = np.angle(vals)
flags = np.array((weights[0] == 0) | (weights[1] == 0), dtype=bool)
# unwrap
if doUnwrap:
# remove mean to facilitate unwrapping
for a, ant in enumerate(coord['ant']):
if not (flags[a, :, :] == True).all() and ant != refAnt:
logging.debug('Unwrapping: ' + ant)
mean = np.angle(np.nanmean(np.exp(1j * vals[a].flatten())))
vals[a] -= mean
vals[a] = np.mod(vals[a] + np.pi, 2 * np.pi) - np.pi
vals[a, :, :] = unwrap_2d(vals[a, :, :], flags[a, :, :],
coord['freq'], coord['time'])
logging.debug('Computing differential values...')
t1 = np.ma.array(vals, mask=flags) # mask flagged data
dph = t1[np.newaxis] - t1[:, np.newaxis] # ant x ant x freq x time
D = pos[np.newaxis] - pos[:, np.newaxis] # ant x ant x 3
D2 = np.triu(
np.sqrt(np.sum(D**2, axis=-1))
) # calc distance and keep only uppoer triangle larger than 0
myselect = (D2 > 0)
if not doUnwrap:
logging.debug('Re-normalising...')
dph = np.mod(dph + np.pi, 2 * np.pi) - np.pi
avgdph = np.ma.average(
dph,
axis=2) # avg in freq (can do because is between -pi and pi)
#one extra step to remove most(all) phase wraps, phase wraps disturbe the averaging...
dph = np.remainder(
dph -
np.ma.average(avgdph, axis=-1)[:, :, np.newaxis, np.newaxis] +
np.pi, 2 * np.pi) + np.ma.average(
avgdph,
axis=-1)[:, :, np.newaxis,
np.newaxis] - np.pi #center around the avg value
logging.debug('Computing sructure function...')
avgdph = np.ma.average(dph, axis=2) # avg in freq to reduce noise
variances = []
pars = []
avgdph = avgdph[
..., avgdph.shape[-1] %
ndiv:] # remove a few timeslots to make the array divisible by np.split
for i, avgdphSplit in enumerate(np.split(avgdph, ndiv, axis=-1)):
variance = np.ma.var(avgdphSplit, axis=-1) * (
np.average(coord['freq']) /
150.e6)**2 # get time variance and rescale to 150 MHz
# linear regression
#A = np.ones((2,D2[myselect].shape[0]),dtype=float)
#A[1,:] = np.log10(D2[myselect][~variance.mask])
#par = np.dot(np.linalg.inv(np.dot(A,A.T)),np.dot(A,np.log10(variance[myselect])))
mask = variance[myselect].mask
A = np.vstack([
np.log10(D2[myselect][~mask]),
np.ones(len(D2[myselect][~mask]))
])
par = np.linalg.lstsq(A.T, np.log10(variance[myselect][~mask]))[0]
S0 = 10**(-1 * par[1] / par[0])
logging.info(r't%i: beta=%.2f - R_diff=%.2f km' %
(i, par[0], S0 / 1.e3))
variances.append(variance)
pars.append(par)
if plotName != '':
if plotName.split('.')[-1] != 'png': plotName += '.png' # add png
if not 'matplotlib' in sys.modules:
import matplotlib as mpl
mpl.use("Agg")
import matplotlib.pyplot as plt
fig = plt.figure()
fig.subplots_adjust(wspace=0)
ax = fig.add_subplot(111)
ax1 = ax.twinx()
for i, variance in enumerate(variances):
if len(variances) > 1:
color = plt.cm.jet(
i / float(len(variances) - 1)) # from 0 to 1
else:
color = 'black'
ax.plot(D2[myselect] / 1.e3,
variance[myselect],
marker='o',
linestyle='',
color=color,
markeredgecolor='none',
label='T')
# regression
par = pars[i]
x = D2[myselect]
S0 = 10**(-1 * par[1] / par[0])
if color == 'black':
color = 'red' # in case of single color, use red line that is more visible
ax1.plot(x.flatten() / 1.e3,
par[0] * np.log10(x.flatten()) + par[1],
linestyle='-',
color=color,
label=r'$\beta=%.2f$ - $R_{\rm diff}=%.2f$ km' %
(par[0], S0 / 1.e3))
ax.set_xlabel('Distance (km)')
ax.set_ylabel(r'Phase variance @150 MHz (rad$^2$)')
ax.set_xscale('log')
ax.set_yscale('log')
ymin = np.min(variance[myselect])
ymax = np.max(variance[myselect])
ax.set_xlim(xmin=0.1, xmax=3)
ax.set_ylim(ymin, ymax)
ax1.set_ylim(np.log10(ymin), np.log10(ymax))
ax1.legend(loc='lower right', frameon=False)
ax1.set_yticks([])
logging.warning('Save pic: %s' % plotName)
plt.savefig(plotName, bbox_inches='tight')
return 0
#!/usr/bin/env python
# -*- coding: utf-8 -*-
from losoto.lib_operations import *
from losoto._logging import logger as logging
logging.debug('Loading STRUCTURE module.')
def _run_parser(soltab, parser, step):
doUnwrap = parser.getbool(step, 'doUnwrap', False)
refAnt = parser.getstr(step, 'refAnt', '')
plotName = parser.getstr(step, 'plotName', '')
ndiv = parser.getint(step, 'ndiv', 1)
parser.checkSpelling(step, soltab,
['doUnwrap', 'refAnt', 'plotName', 'ndiv'])
return run(soltab, doUnwrap, refAnt, plotName, ndiv)
def run(soltab, doUnwrap=False, refAnt='', plotName='', ndiv=1):
"""
Find the structure function from phase solutions of core stations.
Parameters
----------
doUnwrap : bool, optional
refAnt : str, optional
Reference antenna, by default the first.
#!/usr/bin/env python
# -*- coding: utf-8 -*-
from losoto.lib_operations import *
from losoto._logging import logger as logging
logging.debug('Loading REFERENCE module.')
def _run_parser(soltab, parser, step):
refAnt = parser.getstr( step, 'refAnt', '' )
refDir = parser.getstr( step, 'refDir', '' )
parser.checkSpelling( step, soltab, ['refAnt','refDir'])
return run(soltab, refAnt, refDir)
def run( soltab, refAnt='', refDir=''):
"""
Reference to an antenna
Parameters
----------
refAnt : str, optional
Reference antenna for phases or "best" to use the least flagged antenna. Empty string does not change phases. Default: ''.
refDir : str, optional
Reference direction for phases. Empty string does not change phases. Default: ''.
"""
if not soltab.getType() in ['phase', 'scalarphase', 'rotation', 'tec', 'clock', 'tec3rd', 'rotationmeasure']:
logging.error('Reference possible only for phase, scalarphase, clock, tec, tec3rd, rotation and rotationmeasure solution tables. Ignore referencing.')
return 1
#!/usr/bin/env python
# -*- coding: utf-8 -*-
from losoto.lib_operations import *
from losoto._logging import logger as logging
logging.debug('Loading SPLITLEAK module.')
def _run_parser(soltab, parser, step):
soltabOutG = parser.getstr(step, 'soltabOutG') # no default
soltabOutD = parser.getstr(step, 'soltabOutD') # no default
parser.checkSpelling(step, soltab, ['soltabOutG', 'soltabOutD'])
return run(soltab, soltabOutG, soltabOutD)
def run(soltab, soltabOutG=None, soltabOutD=None):
"""
Duplicate a table
Parameters
----------
soltabOutG : str, optional
Output table name (diagonal component). By default choose next available from table type.
soltabOutD : str, optional
Output table name (leakage component). By default choose next available from table type.
"""
logging.info('Split leakage tables %s -> %s + %s' %
#!/usr/bin/env python
# -*- coding: utf-8 -*-
from losoto.lib_operations import *
from losoto._logging import logger as logging
logging.debug('Loading NORM module.')
def _run_parser(soltab, parser, step):
axesToNorm = parser.getarraystr(step, 'axesToNorm') # no default
normVal = parser.getfloat(step, 'normVal', 1.)
log = parser.getbool(step, 'log', False)
parser.checkSpelling(step, soltab, ['axesToNorm', 'normVal', 'log'])
return run(soltab, axesToNorm, normVal, log)
def run(soltab, axesToNorm, normVal=1., log=False):
"""
Normalize the solutions to a given value
WEIGHT: Weights compliant
Parameters
----------
axesToNorm : array of str
Axes along which compute the normalization.
normVal : float, optional
Number to normalize to vals = vals * (normVal/valsMean), by default 1.
#import warnings
#warnings.simplefilter('error', UserWarning)
from losoto.lib_operations import *
from losoto._logging import logger as logging
# avoids error if re-setting "agg" a second run of plot
if not 'matplotlib' in sys.modules:
import matplotlib as mpl
mpl.rcParams['xtick.labelsize'] = 20
mpl.rcParams['font.size'] = 20
mpl.use("Agg")
import matplotlib.pyplot as plt # after setting "Agg" to speed up
logging.debug('Loading PLOT module.')
def _run_parser(soltab, parser, step):
axesInPlot = parser.getarraystr(step, 'axesInPlot') # no default
axisInTable = parser.getstr(step, 'axisInTable', '')
axisInCol = parser.getstr(step, 'axisInCol', '')
axisDiff = parser.getstr(step, 'axisDiff', '')
NColFig = parser.getint(step, 'NColFig', 0)
figSize = parser.getarrayint(step, 'figSize', [0, 0])
markerSize = parser.getint(step, 'markerSize', 2)
minmax = parser.getarrayfloat(step, 'minmax', [0., 0.])
log = parser.getstr(step, 'log', '')
plotFlag = parser.getbool(step, 'plotFlag', False)
doUnwrap = parser.getbool(step, 'doUnwrap', False)
refAnt = parser.getstr(step, 'refAnt', '')
#!/usr/bin/env python
# -*- coding: utf-8 -*-
from losoto.lib_operations import *
from losoto._logging import logger as logging
logging.debug('Loading TEC module.')
def _run_parser(soltab, parser, step):
soltabOut = parser.getstr(step, 'soltabOut', 'tec000')
refAnt = parser.getstr(step, 'refAnt', '')
maxResidualFlag = parser.getfloat(step, 'maxResidualFlag', 2.5)
maxResidualProp = parser.getfloat(step, 'maxResidualProp', 1.)
parser.checkSpelling(
step, soltab,
['soltabOut', 'refAnt', 'maxResidualFlag', 'maxResidualProp'])
return run(soltab, soltabOut, refAnt, maxResidualFlag, maxResidualProp)
def run(soltab,
soltabOut='tec000',
refAnt='',
maxResidualFlag=2.5,
maxResidualProp=1.):
"""
Bruteforce TEC extraction from phase solutions.
Parameters
----------
#!/usr/bin/env python
# -*- coding: utf-8 -*-
# This is the station-screen operation for LoSoTo
from losoto.lib_operations import *
from losoto._logging import logger as logging
logging.debug('Loading STATIONSCREEN module.')
def _run_parser(soltab, parser, step):
outSoltab = parser.getstr(step, "outSoltab")
order = parser.getint(step, "Order", 5)
beta = parser.getfloat(step, "Beta", 5.0 / 3.0)
niter = parser.getint(step, "niter", 2)
nsigma = parser.getfloat(step, "nsigma", 5.0)
refAnt = parser.getint(step, "RefAnt", -1)
scale_order = parser.getbool(step, "ScaleOrder", True)
scale_dist = parser.getfloat(step, "scaleDist", 25000.0)
min_order = parser.getint(step, "MinOrder", 5)
adjust_order = parser.getbool(step, "AdjustOrder", True)
parser.checkSpelling( step, soltab, ['outSoltab', 'order', 'beta', 'niter', 'nsigma',\
'refAnt', 'scale_order', 'scale_dist', 'min_order', 'adjust_order'])
return run(soltab, outSoltab, order, beta, niter, nsigma, refAnt,
scale_order, scale_dist, min_order, adjust_order)
def _calculate_piercepoints(station_positions, source_positions):
"""
#!/usr/bin/env python
# -*- coding: utf-8 -*-
from losoto.lib_operations import *
from losoto._logging import logger as logging
logging.debug('Loading GLOBALDELAY module.')
def _run_parser(soltab, parser, step):
soltabOut = parser.getstr(step, 'soltabOut', 'delay000')
refAnt = parser.getstr(step, 'refAnt', '')
parser.checkSpelling(step, soltab, ['soltabOut', 'refAnt'])
return run(soltab, soltabOut, refAnt)
def run(soltab, soltabOut='tec000', refAnt=''):
"""
Bruteforce TEC extraction from phase solutions.
Parameters
----------
soltabOut : str, optional
output table name (same solset), by deault "tec".
refAnt : str, optional
Reference antenna, by default the first.
"""
import numpy as np
def getClockTECFitStation(
ph,
freq,
stationname,
initSol=[],
returnResiduals=True,
chi2cut=1e8,
fit3rdorder=False,
double_search_space=False
):
'''get the c/t separation per station'''
nT = ph.shape[0]
nF = freq.shape[0]
data = ph
tecarrayst = np.zeros((nT,), dtype=np.float32)
clockarrayst = np.zeros((nT,), dtype=np.float32)
if returnResiduals:
residualarrayst = np.zeros((nT, nF), dtype=np.float32)
if fit3rdorder:
tec3rdarrayst= np.zeros((nT,), dtype=np.float32)
A = np.ones((nF, 2+fit3rdorder), dtype=np.float)
A[:, 1] = freq * 2 * np.pi * 1e-9
A[:, 0] = -8.44797245e9 / freq
if fit3rdorder:
A[:, 2] = -1e21 / freq**3
steps = np.ma.dot(np.ma.dot(np.linalg.inv(np.ma.dot(A.T, A)), A.T), 2 * np.pi * np.ones((freq.shape[0], ), dtype=np.float))
succes=False
initprevsol=False
nrFail=0
sol = np.zeros((2+fit3rdorder), dtype=np.float)
prevsol = np.zeros_like(sol)
n3rd=0
for itm in range(nT):
datatmp=np.ma.copy(data[itm, :])
if itm == 0 or not succes:
if itm == 0 or not initprevsol:
if hasattr(initSol, '__len__') and len(initSol) > 0 :
sol[:initSol.shape[0]]=initSol
ndt=1
ndtec=1
if fit3rdorder:
n3rd=1
else:
sol[:]=0
if fit3rdorder:
n3rd=200
if 'CS' in stationname:
ndt=4
ndtec=10
if 'LBA' in stationname:
ndt=2
ndtec=40
else:
if 'RS' in stationname:
ndt=200
ndtec=80
else:
# large TEC variation for EU stations
ndt=200
ndtec=160
if 'LBA' in stationname:
ndt=60
# no init clock possible due to large TEC effect
#stepsize of dtec is small
ndtec=320
else:
# further steps with non success
sol[:] = prevsol[:]
ndtec=min(nrFail+1,100)
if not 'CS' in stationname:
ndt=min(nrFail+1,200)
else:
ndt=min(nrFail+1,4)
if fit3rdorder:
n3rd=min(nrFail+1,200)
datatmpist = datatmp[:]
if datatmpist.count() / float(nF) > 0.5:
# do brutforce and update data, unwrp pdata,update flags
#logging.debug("Getting init par for time %d:station %s ntec %d ndt %d n3rd %d"%(itm,stationname,ndtec,ndt,n3rd)+str(sol))
par,datatmp[:] = getInitPar(datatmpist, freq,nrTEC=ndtec*(1+double_search_space),nrClock=ndt*(1+double_search_space),nrthird=n3rd*(1+double_search_space),initsol=sol[:])
sol[:] = par[:]
#logging.debug("Getting init par for station %d:%s "%(itm,stationname)+str(sol))
#now do the real fitting
datatmpist=datatmp[:]
if datatmpist.count() / float(nF) < 0.5:
logging.debug("Too many data points flagged 2nd t=%d st=%s flags=%d" % (itm,stationname,datatmpist.count()) + str(sol[:]))
sol[:] = [-10.,]*sol.shape[0]
else:
fitdata=np.dot(sol,A.T)
datatmpist=unwrapPhases(datatmpist,freq,fitdata)
mymask=datatmpist.mask
maskedfreq=np.ma.array(freq,mask=mymask)
A2=np.ma.array(A,mask=np.tile(mymask,(A.shape[1],1)).T)
if datatmpist.count() / float(nF) < 0.5:
logging.debug("Too many data points flagged 3rd t=%d st=%s flags=%d" % (itm,stationname,datatmpist.count()) + str(sol[:]))
sol[:] = [-10.,]*sol.shape[0]
else:
sol[:] = np.ma.dot(np.linalg.inv(np.ma.dot(A2.T, A2)), np.ma.dot(A2.T, datatmpist)).T
if initprevsol and np.abs((sol[1]-prevsol[1])/steps[1])>0.5 and (np.abs((sol[1]-prevsol[1])/steps[1])>0.75 or np.abs(np.sum((sol-prevsol)/steps,axis=-1))>0.5*A2.shape[0]):
#logging.debug("removing jumps, par for station %d , itm %d"%(ist,itm)+str(sol[ist])+str(prevsol[ist])+str(steps))
sol[:]-=np.round((sol[1]-prevsol[1])/steps[1])*steps
# calculate chi2 per station
residual = data[itm] - np.dot(A, sol)
residual = np.ma.remainder(residual + np.pi, 2 * np.pi) - np.pi
chi2 = np.ma.sum(np.square(np.degrees(residual)), axis=0) / nF
if returnResiduals:
residualarrayst[itm] = residual
chi2select = (chi2 > chi2cut) or (sol[0] < -5) # select bad points
chi2select = chi2select or initprevsol*np.sum(np.abs((sol-prevsol)/steps),axis=-1)>(0.3*sol.shape[0]*(1+nrFail)) #also discard data where there is a "half" jump for any parameter wrst the previous solution (if previous solution exists). Multiply with number of fails, since after a large number of fails there is no clear match with the previous solution expected anyway...
if not initprevsol:
prevsol= sol[:]
if chi2select:
logging.debug('High chi2 of fit, itm: %d ' % (itm) + 'station:' + stationname)
succes = False
nrFail += 1
else:
prevsol = 0.5 * prevsol + 0.5 * np.copy(sol)
succes = True
initprevsol = True
nrFail = 0
tecarrayst[itm] = sol[0]
clockarrayst[itm] = sol[1]
if fit3rdorder:
tec3rdarrayst[itm]=sol[2]
if returnResiduals:
if fit3rdorder:
return (tecarrayst, clockarrayst, residualarrayst,tec3rdarrayst)
else:
return (tecarrayst, clockarrayst, residualarrayst)
if fit3rdorder:
return (tecarrayst, clockarrayst,tec3rdarrayst)
return (tecarrayst, clockarrayst)
def getClockTECFit(
ph,
freq,
stations,
initSol=[],
returnResiduals=True,
chi2cut=1e8,
fit3rdorder=False,
double_search_space=False
):
nT = ph.shape[0]
nF = freq.shape[0]
nSt = ph.shape[2]
data = ph
tecarray = np.zeros((nT, nSt), dtype=np.float32)
clockarray = np.zeros((nT, nSt), dtype=np.float32)
if returnResiduals:
residualarray = np.zeros((nT, nF, nSt), dtype=np.float32)
if fit3rdorder:
tec3rdarray= np.zeros((nT, nSt), dtype=np.float32)
A = np.ones((nF, 2+fit3rdorder), dtype=np.float)
A[:, 1] = freq * 2 * np.pi * 1e-9
A[:, 0] = -8.44797245e9 / freq
if fit3rdorder:
A[:, 2] = -1e21 / freq**3
steps = np.ma.dot(np.ma.dot(np.linalg.inv(np.ma.dot(A.T, A)), A.T), 2 * np.pi * np.ones((freq.shape[0], ), dtype=np.float))
succes=False
initprevsol=np.zeros(nSt,dtype=bool)
nrFail=np.zeros(nSt,dtype=int)
sol = np.zeros((nSt, 2+fit3rdorder), dtype=np.float)
prevsol = np.zeros_like(sol)
n3rd=0
for itm in range(nT):
datatmp=np.ma.copy(data[itm, :])
if itm == 0 or not succes:
for ist in range(nSt):
if itm == 0 or not initprevsol[ist]:
if hasattr(initSol, '__len__') and len(initSol) > ist:
sol[ist,:initSol[ist].shape[0]]=initSol[ist]
ndt=1
ndtec=1
if fit3rdorder:
n3rd=1
else:
sol[ist]=0
if fit3rdorder:
n3rd=200
if 'CS' in stations[ist]:
ndt=4
ndtec=10
if 'LBA' in stations[ist]:
ndt=2
ndtec=40
else:
if 'RS' in stations[ist]:
ndt=200
ndtec=80
else:
# large TEC variation for EU stations
ndt=200
ndtec=160
if 'LBA' in stations[ist]:
ndt=60
# no init clock possible due to large TEC effect
#stepsize of dtec is small
ndtec=520
else:
# further steps with non success
sol[ist, :] = prevsol[ist, :]
ndtec=min(nrFail[ist]+1,100)
if not 'CS' in stations[ist]:
ndt=min(nrFail[ist]+1,200)
else:
ndt=min(nrFail[ist]+1,4)
if fit3rdorder:
n3rd=min(nrFail[ist]+1,200)
datatmpist = datatmp[:, ist]
if datatmpist.count() / float(nF) > 0.5:
# do brutforce and update data, unwrp pdata,update flags
#if ist==23 or ist==25:
#logging.debug("Getting init par for time %d:station %d ntec %d ndt %d n3rd %d"%(itm,ist,ndtec,ndt,n3rd)+str(sol[ist]))
par,datatmp[:, ist] = getInitPar(datatmpist, freq,nrTEC=ndtec*(1+double_search_space),nrClock=ndt*(1+double_search_space),nrthird=n3rd*(1+double_search_space),initsol=sol[ist,:])
sol[ist, :] = par[:]
#if itm%100==0:
#logging.debug("Getting init par for station %d:%d "%(itm,ist)+str(sol[ist]))
for ist in range(nSt):
#now do the real fitting
datatmpist=datatmp[:,ist]
if datatmpist.count() / float(nF) < 0.5:
logging.debug("Too many data points flagged first t=%d st=%d flags=%d" % (itm,ist,datatmpist.count()) + str(sol[ist]))
sol[ist] = [-10.,]*sol.shape[1]
continue
fitdata=np.dot(sol[ist],A.T)
datatmpist=unwrapPhases(datatmpist,freq,fitdata)
#if itm%100==0:
#logging.debug(" init par for station itm %d:%d "%(itm,ist)+str(sol[ist]))
mymask=datatmpist.mask
maskedfreq=np.ma.array(freq,mask=mymask)
A2=np.ma.array(A,mask=np.tile(mymask,(A.shape[1],1)).T)
sol[ist] = np.ma.dot(np.linalg.inv(np.ma.dot(A2.T, A2)), np.ma.dot(A2.T, datatmpist)).T
if initprevsol[ist] and np.abs((sol[ist,1]-prevsol[ist,1])/steps[1])>0.5 and (np.abs((sol[ist,1]-prevsol[ist,1])/steps[1])>0.75 or np.abs(np.sum((sol[ist]-prevsol[ist])/steps,axis=-1))>0.5*A2.shape[0]):
#logging.debug("removing jumps, par for station %d , itm %d"%(ist,itm)+str(sol[ist])+str(prevsol[ist])+str(steps))
sol[ist,:]-=np.round((sol[ist,1]-prevsol[ist,1])/steps[1])*steps
#logging.debug("removed jumps, par for station %d "%ist+str(sol[ist])+str(prevsol[ist]))
#if itm%100==0:
#logging.debug("par for station itm %d:%d "%(itm,ist)+str(sol[ist]))
# calculate chi2 per station
residual = data[itm] - np.dot(A, sol.T)
tmpresid = residual - residual[:, 0][:, np.newaxis] # residuals relative to station 0
residual = np.ma.remainder(tmpresid + np.pi, 2 * np.pi) - np.pi
chi2 = np.ma.sum(np.square(np.degrees(residual)), axis=0) / nF
if returnResiduals:
residualarray[itm] = residual
chi2select = np.logical_or(np.array(chi2 > chi2cut), sol[:, 0] < -5) # select bad points
chi2select = np.logical_or(chi2select,initprevsol*np.sum(np.abs((sol-prevsol)/steps),axis=-1)>(0.3*sol.shape[1]*(1+nrFail))) #also discard data where there is a "half" jump for any parameter wrst the previous solution (if previous solution exists). Multiply with number of fails, since after a large number of fails there is no clear match with the previous solution expected anyway...
if np.any(chi2select):
#logging.debug('high chi2 of fit, itm: %d %d ' % (itm, np.sum(chi2select)) + str(sol[chi2select]) + 'prevsol'+str(prevsol[chi2select])+'stations:' + str(np.arange(nSt)[chi2select]) + ' chi2 ' + str(chi2[chi2select]))
logging.debug('High chi2 of fit, itm: %d %d ' % (itm, np.sum(chi2select)) + 'stations:' + str(np.arange(nSt)[chi2select]))
succes = False
nrFail[chi2select] += 1
nrFail[~chi2select] = 0
prevsol[np.logical_and(initprevsol == False,chi2select==False)] = sol[np.logical_and(initprevsol == False,chi2select==False)] # compensate missing prevsol at first rounds
prevsol[~chi2select] = 0.5 * prevsol[~chi2select] + 0.5 * sol[~chi2select] # init solution to 0.5 * this solution + 0.5 previous solution
initprevsol[~chi2select] = True # once is True it never becomes False
else:
# prevsol=np.copy(sol)
prevsol[initprevsol==False] = sol[initprevsol == False] # compensate missing prevsol at first rounds
prevsol = 0.5 * prevsol + 0.5 * np.copy(sol)
succes = True
initprevsol = np.ones(nSt, dtype=bool)
nrFail = np.zeros(sol.shape[0], dtype=int)
tecarray[itm] = sol[:, 0]
clockarray[itm] = sol[:, 1]
if fit3rdorder:
tec3rdarray[itm]=sol[:, 2]
if returnResiduals:
if fit3rdorder:
return (tecarray, clockarray, residualarray,tec3rdarray)
else:
return (tecarray, clockarray, residualarray)
if fit3rdorder:
return (tecarray, clockarray,tec3rdarray)
return (tecarray, clockarray)
#!/usr/bin/python
# -*- coding: utf-8 -*-
import numpy as np
import numpy.ma as ma
import sys
from multiprocessing import Pool
from losoto._logging import logger as logging
has_fitting=True
try:
import casacore.tables # must be loaded before expion - used in other operations as lofarbeam
from lofar.expion import baselinefitting as fitting
except:
# logging.error('No lofar.expion present, clock/tec separation not active.')
logging.debug('No lofar.expion present, clock/tec separation maybe not optimal')
has_fitting=False
light_speed=299792458.
# from pylab import *
def ClockTEC_3rdorder_func(xarray, par):
"""clock tec fitting withextra parameter for third order ionospheric effects at lowest frequencies"""
delay = np.array([par[1] * 1e-9]).flatten() # in ns, array has dimension 1, even if scalar
delayfact = 2 * np.pi * delay[:, np.newaxis] * xarray
TEC = np.array([par[0]]).flatten() # dTEC in TECU
drefract = -8.4479745e9 * TEC[:, np.newaxis] / xarray
TEC3rd = np.array([par[2]]).flatten() # 3rd_order
d3rd = light_speed**3* TEC3rd[:, np.newaxis] / xarray**3
return drefract[:, np.newaxis,np.newaxis, :] + delayfact[np.newaxis,:,np.newaxis, :]+d3rd[np.newaxis,np.newaxis,:, :]
def ClockTECfunc(xarray, par):
#!/usr/bin/env python
# -*- coding: utf-8 -*-
# This is an example operation for LoSoTo
from losoto.lib_operations import *
from losoto._logging import logger as logging
logging.debug('Loading CLOCKTEC module.')
def _run_parser(soltab, parser, step):
clocksoltabOut = parser.getstr(step, 'clocksoltabOut', 'clock000')
tecsoltabOut = parser.getstr(step, 'tecsoltabOut', 'tec000')
offsetsoltabOut = parser.getstr(step, 'offsetsoltabOut', 'phase_offset000')
tec3rdsoltabOut = parser.getstr(step, 'tec3rdsoltabOut', 'tec3rd000')
flagBadChannels = parser.getbool(step, 'flagBadChannels', True)
flagCut = parser.getfloat(step, 'flagCut', 5.)
chi2cut = parser.getfloat(step, 'chi2cut', 3000.)
combinePol = parser.getbool(step, 'combinePol', False)
removePhaseWraps = parser.getbool(step, 'removePhaseWraps', True)
fit3rdorder = parser.getbool(step, 'fit3rdorder', False)
circular = parser.getbool(step, 'circular', False)
reverse = parser.getbool(step, 'reverse', False)
invertOffset = parser.getbool(step, 'invertOffset', False)
nproc = parser.getint(step, 'nproc', 10)
parser.checkSpelling(step, soltab, [
'tecsoltabOut', 'clocksoltabOut', 'offsetsoltabOut', 'tec3rdsoltabOut',
'flagBadChannels', 'flagCut', 'chi2cut', 'combinePol',
'removePhaseWraps', 'fit3rdorder', 'circular', 'reverse',
def run( soltab, soltabOut='phasediff', maxResidual=1., fitOffset=False, average=False, replace=False, minFreq=0, refAnt='' ):
"""
Estimate polarization misalignment as delay.
Parameters
----------
soltabOut : str, optional
output table name (same solset), by deault "phasediff".
maxResidual : float, optional
Maximum acceptable rms of the residuals in radians before flagging, by default 1. If 0: No check.
fitOffset : bool, optional
Assume that together with a delay each station has also a differential phase offset (important for old LBA observations). By default False.
average : bool, optional
Mean-average in time the resulting delays/offset. By default False.
replace : bool, optional
replace using smoothed value instead of flag bad data? Smooth must be active. By default, False.
minFreq : float, optional
minimum frequency [Hz] to use in estimating the PA. By default, 0 (all freqs).
refAnt : str, optional
Reference antenna, by default the first.
"""
import numpy as np
import scipy.optimize
from scipy import stats
from scipy.ndimage import generic_filter
logging.info("Finding polarization align for soltab: "+soltab.name)
delaycomplex = lambda d, freq, y: abs(np.cos(d[0]*freq) - np.cos(y)) + abs(np.sin(d[0]*freq) - np.sin(y))
#delaycomplex = lambda d, freq, y: abs(d[0]*freq - y)
solType = soltab.getType()
if solType != 'phase':
logging.warning("Soltab type of "+soltab.name+" is of type "+solType+", should be phase. Ignoring.")
return 1
if refAnt != '' and refAnt != 'closest' and not refAnt in soltab.getAxisValues('ant', ignoreSelection = True):
logging.warning('Reference antenna '+refAnt+' not found. Using: '+soltab.getAxisValues('ant')[1])
refAnt = soltab.getAxisValues('ant')[1]
if refAnt == '': refAnt = soltab.getAxisValues('ant')[1]
# times and ants needs to be complete or selection is much slower
times = soltab.getAxisValues('time')
# create new table
solset = soltab.getSolset()
soltabout = solset.makeSoltab(soltype = soltab.getType(), soltabName = soltabOut, axesNames=soltab.getAxesNames(),
axesVals=[soltab.getAxisValues(axisName) for axisName in soltab.getAxesNames()],
vals=soltab.getValues(retAxesVals = False), weights=soltab.getValues(weight = True, retAxesVals = False))
soltabout.addHistory('Created by POLALIGN operation from %s.' % soltab.name)
if 'XX' in soltab.getAxisValues('pol'): pol = 'XX'
elif 'RR' in soltab.getAxisValues('pol'): pol = 'RR'
else:
logging.error('Cannot reference to known polarisation.')
return 1
for vals, weights, coord, selection in soltab.getValuesIter(returnAxes=['freq','pol','time'], weight=True, refAnt=refAnt):
# reorder axes
vals = reorderAxes( vals, soltab.getAxesNames(), ['pol','freq','time'] )
weights = reorderAxes( weights, soltab.getAxesNames(), ['pol','freq','time'] )
if 'RR' in coord['pol'] and 'LL' in coord['pol']:
coord1 = np.where(coord['pol'] == 'RR')[0][0]
coord2 = np.where(coord['pol'] == 'LL')[0][0]
elif 'XX' in coord['pol'] and 'YY' in coord['pol']:
coord1 = np.where(coord['pol'] == 'XX')[0][0]
coord2 = np.where(coord['pol'] == 'YY')[0][0]
if (weights == 0.).all() == True:
logging.warning('Skipping flagged antenna: '+coord['ant'])
weights[:] = 0
else:
fit_delays=[]; fit_offset=[]; fit_weights=[]
for t, time in enumerate(times):
# apply flags
idx = ( (weights[coord1,:,t] != 0.) & (weights[coord2,:,t] != 0.) & (coord['freq'] > minFreq) )
freq = np.copy(coord['freq'])[idx]
phase1 = vals[coord1,:,t][idx]
phase2 = vals[coord2,:,t][idx]
if len(freq) < 30:
fit_weights.append(0.)
fit_delays.append(0.)
fit_offset.append(0.)
logging.debug('Not enough unflagged point for the timeslot '+str(t))
continue
# if more than 1/2 of chans are flagged
if (len(idx) - len(freq))/float(len(idx)) > 1/2.:
logging.debug('High number of filtered out data points for the timeslot %i: %i/%i' % (t, len(idx) - len(freq), len(idx)) )
phase_diff = phase1 - phase2
phase_diff = np.mod(phase_diff + np.pi, 2.*np.pi) - np.pi
phase_diff = np.unwrap(phase_diff)
A = np.vstack([freq, np.ones(len(freq))]).T
fitresultdelay = np.linalg.lstsq(A, phase_diff.T)[0]
# get the closest n*(2pi) to the intercept and refit with only 1 parameter
if not fitOffset:
numjumps = np.around(fitresultdelay[1]/(2*np.pi))
A = np.reshape(freq, (-1,1)) # no b
phase_diff -= numjumps * 2 * np.pi
fitresultdelay = np.linalg.lstsq(A, phase_diff.T)[0]
fitresultdelay = [fitresultdelay[0],0.] # set offset to 0 to keep the rest of the script equal
# fractional residual
residual = np.mean(np.abs( fitresultdelay[0]*freq + fitresultdelay[1] - phase_diff ))
fit_delays.append(fitresultdelay[0])
fit_offset.append(fitresultdelay[1])
if maxResidual == 0 or residual < maxResidual:
fit_weights.append(1.)
else:
# high residual, flag
logging.debug('Bad solution for ant: '+coord['ant']+' (time: '+str(t)+', residual: '+str(residual)+') -> ignoring.')
fit_weights.append(0.)
# Debug plot
doplot = False
#if doplot and t%100==0 and (coord['ant'] == 'RS310LBA' or coord['ant'] == 'CS301LBA'):
if doplot and t%10==0 and (coord['ant'] == 'W04'):
if not 'matplotlib' in sys.modules:
import matplotlib as mpl
mpl.rc('figure.subplot',left=0.05, bottom=0.05, right=0.95, top=0.95,wspace=0.22, hspace=0.22 )
mpl.use("Agg")
import matplotlib.pyplot as plt
fig = plt.figure()
fig.subplots_adjust(wspace=0)
ax = fig.add_subplot(111)
# plot rm fit
plotdelay = lambda delay, offset, freq: np.mod( delay*freq + offset + np.pi, 2.*np.pi) - np.pi
ax.plot(freq, fitresultdelay[0]*freq + fitresultdelay[1], "-", color='purple', label=r'delay:%f$\nu$ (ns) + %f ' % (fitresultdelay[0]*1e9,fitresultdelay[1]) )
ax.plot(freq, np.mod(phase1 + np.pi, 2.*np.pi) - np.pi, 'ob' )
ax.plot(freq, np.mod(phase2 + np.pi, 2.*np.pi) - np.pi, 'og' )
#ax.plot(freq, np.mod(phase_diff + np.pi, 2.*np.pi) - np.pi, '.', color='purple' )
ax.plot(freq, phase_diff, '.', color='purple' )
residual = np.mod(plotdelay(fitresultdelay[0], fitresultdelay[1], freq)-phase_diff + np.pi,2.*np.pi)-np.pi
ax.plot(freq, residual, '.', color='yellow')
ax.set_xlabel('freq')
ax.set_ylabel('phase')
#ax.set_ylim(ymin=-np.pi, ymax=np.pi)
logging.warning('Save pic: '+str(t)+'_'+coord['ant']+'.png')
fig.legend(loc='upper left')
plt.savefig(coord['ant']+'_'+str(t)+'.png', bbox_inches='tight')
del fig
# end cycle in time
fit_weights = np.array(fit_weights)
fit_delays = np.array(fit_delays)
fit_offset = np.array(fit_offset)
# avg in time
if average:
fit_delays_bkp = fit_delays[ fit_weights == 0 ]
fit_offset_bkp = fit_offset[ fit_weights == 0 ]
np.putmask(fit_delays, fit_weights == 0, np.nan)
np.putmask(fit_offset, fit_weights == 0, np.nan)
fit_delays[:] = np.nanmean(fit_delays)
# angle mean
fit_offset[:] = np.angle( np.nansum( np.exp(1j*fit_offset) ) / np.count_nonzero(~np.isnan(fit_offset)) )
if replace:
fit_weights[ fit_weights == 0 ] = 1.
fit_weights[ np.isnan(fit_delays) ] = 0. # all the size was flagged cannot estrapolate value
else:
fit_delays[ fit_weights == 0 ] = fit_delays_bkp
fit_offset[ fit_weights == 0 ] = fit_offset_bkp
logging.info('%s: average delay: %f ns (offset: %f)' % ( coord['ant'], np.mean(fit_delays)*1e9, np.mean(fit_offset)))
for t, time in enumerate(times):
#vals[:,:,t] = 0.
#vals[coord1,:,t] = fit_delays[t]*np.array(coord['freq'])/2.
vals[coord1,:,t] = 0
phase = np.mod(fit_delays[t]*coord['freq'] + fit_offset[t] + np.pi, 2.*np.pi) - np.pi
vals[coord2,:,t] = -1.*phase#/2.
#weights[:,:,t] = 0.
weights[coord1,:,t] = fit_weights[t]
weights[coord2,:,t] = fit_weights[t]
# reorder axes back to the original order, needed for setValues
vals = reorderAxes( vals, ['pol','freq','time'], [ax for ax in soltab.getAxesNames() if ax in ['pol','freq','time']] )
weights = reorderAxes( weights, ['pol','freq','time'], [ax for ax in soltab.getAxesNames() if ax in ['pol','freq','time']] )
soltabout.setSelection(**coord)
soltabout.setValues( vals )
soltabout.setValues( weights, weight=True )
return 0
def run(soltab, axesInPlot, axisInTable='', axisInCol='', axisDiff='', NColFig=0, figSize=[0,0], markerSize=2, minmax=[0,0], log='', \
plotFlag=False, doUnwrap=False, refAnt='', soltabsToAdd='', makeAntPlot=False, makeMovie=False, prefix='', ncpu=0):
"""
This operation for LoSoTo implements basic plotting
WEIGHT: flag-only compliant, no need for weight
Parameters
----------
axesInPlot : array of str
1- or 2-element array which says the coordinates to plot (2 for 3D plots).
axisInTable : str, optional
the axis to plot on a page - e.g. ant to get all antenna's on one file. By default ''.
axisInCol : str, optional
The axis to plot in different colours - e.g. pol to get correlations with different colors. By default ''.
axisDiff : str, optional
This must be a len=2 axis and the plot will have the differential value - e.g. 'pol' to plot XX-YY. By default ''.
NColFig : int, optional
Number of columns in a multi-table image. By default is automatically chosen.
figSize : array of int, optional
Size of the image [x,y], if one of the values is 0, then it is automatically chosen. By default automatic set.
markerSize : int, optional
Size of the markers in the 2D plot. By default 2.
minmax : array of float, optional
Min max value for the independent variable (0 means automatic). By default 0.
log : bool, optional
Use Log='XYZ' to set which axes to put in Log. By default ''.
plotFlag : bool, optional
Whether to plot also flags as red points in 2D plots. By default False.
doUnwrap : bool, optional
Unwrap phases. By default False.
refAnt : str, optional
Reference antenna for phases. By default None.
soltabsToAdd : str, optional
Tables to "add" (e.g. 'sol000/tec000'), it works only for tec and clock to be added to phases. By default None.
makeAntPlot : bool, optional
Make a plot containing antenna coordinates in x,y and in color the value to plot, axesInPlot must be [ant]. By default False.
makeMovie : bool, optional
Make a movie summing up all the produced plots, by default False.
prefix : str, optional
Prefix to add before the self-generated filename, by default None.
ncpu : int, optional
Number of cpus, by default all available.
"""
import os, random
import numpy as np
from losoto.lib_unwrap import unwrap, unwrap_2d
logging.info("Plotting soltab: " + soltab.name)
# input check
# str2list
if axisInTable == '': axisInTable = []
else: axisInTable = [axisInTable]
if axisInCol == '': axisInCol = []
else: axisInCol = [axisInCol]
if axisDiff == '': axisDiff = []
else: axisDiff = [axisDiff]
if len(set(axisInTable + axesInPlot + axisInCol +
axisDiff)) != len(axisInTable + axesInPlot + axisInCol +
axisDiff):
logging.error('Axis defined multiple times.')
return 1
# just because we use lists, check that they are 1-d
if len(axisInTable) > 1 or len(axisInCol) > 1 or len(axisDiff) > 1:
logging.error(
'Too many TableAxis/ColAxis/DiffAxis, they must be at most one each.'
)
return 1
for axis in axesInPlot + axisInCol + axisDiff:
if axis not in soltab.getAxesNames():
logging.error('Axis \"' + axis + '\" not found.')
return 1
if makeMovie:
prefix = prefix + '__tmp__'
if os.path.dirname(prefix) != '' and not os.path.exists(
os.path.dirname(prefix)):
logging.debug('Creating ' + os.path.dirname(prefix) + '.')
os.makedirs(os.path.dirname(prefix))
if refAnt == '': refAnt = None
elif refAnt != 'closest' and not refAnt in soltab.getAxisValues(
'ant', ignoreSelection=True):
logging.error('Reference antenna ' + refAnt + ' not found. Using: ' +
soltab.getAxisValues('ant')[1])
refAnt = soltab.getAxisValues('ant')[1]
minZ, maxZ = minmax
solset = soltab.getSolset()
soltabsToAdd = [
solset.getSoltab(soltabName) for soltabName in soltabsToAdd
]
cmesh = False
if len(axesInPlot) == 2:
cmesh = True
# not color possible in 3D
axisInCol = []
elif len(axesInPlot) != 1:
logging.error('Axes must be a len 1 or 2 array.')
return 1
# end input check
# all axes that are not iterated by anything else
axesInFile = soltab.getAxesNames()
for axis in axisInTable + axesInPlot + axisInCol + axisDiff:
axesInFile.remove(axis)
# set subplots scheme
if axisInTable != []:
Nplots = soltab.getAxisLen(axisInTable[0])
else:
Nplots = 1
# prepare antennas coord in makeAntPlot case
if makeAntPlot:
if axesInPlot != ['ant']:
logging.error(
'If makeAntPlot is selected the "Axes" values must be "ant"')
return 1
antCoords = [[], []]
for ant in soltab.getAxisValues(
'ant'): # select only user-selected antenna in proper order
antCoords[0].append(+1 * soltab.getSolset().getAnt()[ant][1])
antCoords[1].append(-1 * soltab.getSolset().getAnt()[ant][0])
else:
antCoords = []
datatype = soltab.getType()
# start processes for multi-thread
mpm = multiprocManager(ncpu, _plot)
# compute dataCube size
shape = []
if axisInTable != []: shape.append(soltab.getAxisLen(axisInTable[0]))
else: shape.append(1)
if axisInCol != []: shape.append(soltab.getAxisLen(axisInCol[0]))
else: shape.append(1)
if cmesh:
shape.append(soltab.getAxisLen(axesInPlot[1]))
shape.append(soltab.getAxisLen(axesInPlot[0]))
else:
shape.append(soltab.getAxisLen(axesInPlot[0]))
# will contain the data to pass to each thread to make 1 image
dataCube = np.ma.zeros(shape=shape, fill_value=np.nan)
# cycle on files
if makeMovie: pngs = [] # store png filenames
for vals, coord, selection in soltab.getValuesIter(
returnAxes=axisDiff + axisInTable + axisInCol + axesInPlot):
# set filename
filename = ''
for axis in axesInFile:
filename += axis + str(coord[axis]) + '_'
filename = filename[:-1] # remove last _
if prefix + filename == '': filename = 'plot'
# axis vals (they are always the same, regulat arrays)
xvals = coord[axesInPlot[0]]
# if plotting antenna - convert to number
if axesInPlot[0] == 'ant':
xvals = np.arange(len(xvals))
# if plotting time - convert in h/min/s
xlabelunit = ''
if axesInPlot[0] == 'time':
if xvals[-1] - xvals[0] > 3600:
xvals = (xvals - xvals[0]) / 3600. # hrs
xlabelunit = ' [hr]'
elif xvals[-1] - xvals[0] > 60:
xvals = (xvals - xvals[0]) / 60. # mins
xlabelunit = ' [min]'
else:
xvals = (xvals - xvals[0]) # sec
xlabelunit = ' [s]'
# if plotting freq convert in MHz
elif axesInPlot[0] == 'freq':
xvals = xvals / 1.e6 # MHz
xlabelunit = ' [MHz]'
if cmesh:
# axis vals (they are always the same, regular arrays)
yvals = coord[axesInPlot[1]]
# same as above but for y-axis
if axesInPlot[1] == 'ant':
yvals = np.arange(len(yvals))
if len(xvals) <= 1 or len(yvals) <= 1:
logging.error(
'3D plot must have more then one value per axes.')
mpm.wait()
return 1
ylabelunit = ''
if axesInPlot[1] == 'time':
if yvals[-1] - yvals[0] > 3600:
yvals = (yvals - yvals[0]) / 3600. # hrs
ylabelunit = ' [hr]'
elif yvals[-1] - yvals[0] > 60:
yvals = (yvals - yvals[0]) / 60. # mins
ylabelunit = ' [min]'
else:
yvals = (yvals - yvals[0]) # sec
ylabelunit = ' [s]'
elif axesInPlot[1] == 'freq': # Mhz
yvals = yvals / 1.e6
ylabelunit = ' [MHz]'
else:
yvals = None
if datatype == 'clock':
datatype = 'Clock'
ylabelunit = ' (s)'
elif datatype == 'tec':
datatype = 'dTEC'
ylabelunit = ' (TECU)'
elif datatype == 'rotationmeasure':
datatype = 'dRM'
ylabelunit = r' (rad m$^{-2}$)'
elif datatype == 'tec3rd':
datatype = r'dTEC$_3$'
ylabelunit = r' (rad m$^{-3}$)'
else:
ylabelunit = ''
# cycle on tables
soltab1Selection = soltab.selection # save global selection and subselect only axex to iterate
soltab.selection = selection
titles = []
for Ntab, (vals, coord, selection) in enumerate(
soltab.getValuesIter(returnAxes=axisDiff + axisInCol +
axesInPlot)):
# set tile
titles.append('')
for axis in coord:
if axis in axesInFile + axesInPlot + axisInCol: continue
titles[Ntab] += axis + ':' + str(coord[axis]) + ' '
titles[Ntab] = titles[Ntab][:-1] # remove last ' '
# cycle on colors
soltab2Selection = soltab.selection
soltab.selection = selection
for Ncol, (vals, weight, coord, selection) in enumerate(
soltab.getValuesIter(returnAxes=axisDiff + axesInPlot,
weight=True,
reference=refAnt)):
# differential plot
if axisDiff != []:
# find ordered list of axis
names = [
axis for axis in soltab.getAxesNames()
if axis in axisDiff + axesInPlot
]
if axisDiff[0] not in names:
logging.error("Axis to differentiate (%s) not found." %
axisDiff[0])
mpm.wait()
return 1
if len(coord[axisDiff[0]]) != 2:
logging.error(
"Axis to differentiate (%s) has too many values, only 2 is allowed."
% axisDiff[0])
mpm.wait()
return 1
# find position of interesting axis
diff_idx = names.index(axisDiff[0])
# roll to first place
vals = np.rollaxis(vals, diff_idx, 0)
vals = vals[0] - vals[1]
weight = np.rollaxis(weight, diff_idx, 0)
weight[0][weight[1] == 0] = 0
weight = weight[0]
del coord[axisDiff[0]]
# add tables if required (e.g. phase/tec)
for soltabToAdd in soltabsToAdd:
logging.warning('soltabsToAdd not implemented. Ignoring.')
# newCoord = {}
# for axisName in coord.keys():
# # prepare selected on present axes
# if axisName in soltabToAdd.getAxesNames():
# if type(coord[axisName]) is np.ndarray:
# newCoord[axisName] = coord[axisName]
# else:
# newCoord[axisName] = [coord[axisName]] # avoid being interpreted as regexp, faster
#
# soltabToAdd.setSelection(**newCoord)
# valsAdd = np.squeeze(soltabToAdd.getValues(retAxesVals=False, weight=False, reference=refAnt))
#
# # add missing axes
# print ('shape:', vals.shape)
# for axisName in coord.keys():
# if not axisName in soltabToAdd.getAxesNames():
# # find axis positions
# axisPos = soltab.getAxesNames().index(axisName)
# # create a new axes for the table to add and duplicate the values
# valsAdd = np.expand_dims(valsAdd, axisPos)
# print ('shape to add:', valsAdd.shape)
#
# if soltabToAdd.getType() == 'clock':
# valsAdd = 2. * np.pi * valsAdd * coord['freq']
# elif soltabToAdd.getType() == 'tec':
# valsAdd = -8.44797245e9 * valsAdd / coord['freq']
# else:
# logging.warning('Only Clock or TEC can be added to solutions. Ignoring: '+soltabToAdd.getType()+'.')
# continue
#
# if valsAdd.shape != vals.shape:
# logging.error('Cannot combine the table '+soltabToAdd.getType()+' with '+soltab.getType()+'. Wrong shape.')
# mpm.wait()
# return 1
#
# vals += valsAdd
# normalize
if (soltab.getType() == 'phase'
or soltab.getType() == 'scalarphase'):
vals = normalize_phase(vals)
if (soltab.getType() == 'rotation'):
vals = np.mod(vals + np.pi / 2., np.pi) - np.pi / 2.
# is user requested axis in an order that is different from h5parm, we need to transpose
if cmesh:
if soltab.getAxesNames().index(
axesInPlot[0]) < soltab.getAxesNames().index(
axesInPlot[1]):
vals = vals.T
weight = weight.T
# unwrap if required
if (soltab.getType() == 'phase'
or soltab.getType() == 'scalarphase') and doUnwrap:
if len(axesInPlot) == 1:
vals = unwrap(vals)
else:
flags = np.array((weight == 0), dtype=bool)
if not (flags == True).all():
vals = unwrap_2d(vals, flags, coord[axesInPlot[0]],
coord[axesInPlot[1]])
dataCube[Ntab, Ncol] = vals
sel1 = np.where(weight == 0.)
sel2 = np.where(np.isnan(vals))
if cmesh:
dataCube[Ntab, Ncol, sel1[0], sel1[1]] = np.ma.masked
dataCube[Ntab, Ncol, sel2[0], sel2[1]] = np.ma.masked
else:
dataCube[Ntab, Ncol, sel1[0]] = np.ma.masked
dataCube[Ntab, Ncol, sel2[0]] = np.ma.masked
soltab.selection = soltab2Selection
### end cycle on colors
# if dataCube too large (> 500 MB) do not go parallel
if np.array(dataCube).nbytes > 1024 * 1024 * 500:
logging.debug('Big plot, parallel not possible.')
_plot(Nplots, NColFig, figSize, markerSize, cmesh, axesInPlot,
axisInTable, xvals, yvals, xlabelunit, ylabelunit, datatype,
prefix + filename, titles, log, dataCube, minZ, maxZ,
plotFlag, makeMovie, antCoords, None)
else:
mpm.put([
Nplots, NColFig, figSize, markerSize, cmesh, axesInPlot,
axisInTable, xvals, yvals, xlabelunit, ylabelunit, datatype,
prefix + filename, titles, log,
np.ma.copy(dataCube), minZ, maxZ, plotFlag, makeMovie,
antCoords
])
if makeMovie: pngs.append(prefix + filename + '.png')
soltab.selection = soltab1Selection
### end cycle on tables
mpm.wait()
if makeMovie:
def long_substr(strings):
"""
Find longest common substring
"""
substr = ''
if len(strings) > 1 and len(strings[0]) > 0:
for i in range(len(strings[0])):
for j in range(len(strings[0]) - i + 1):
if j > len(substr) and all(strings[0][i:i + j] in x
for x in strings):
substr = strings[0][i:i + j]
return substr
movieName = long_substr(pngs)
assert movieName != '' # need a common prefix, use prefix keyword in case
logging.info('Making movie: ' + movieName)
# make every movie last 20 sec, min one second per slide
fps = np.ceil(len(pngs) / 200.)
ss="mencoder -ovc lavc -lavcopts vcodec=mpeg4:vpass=1:vbitrate=6160000:mbd=2:keyint=132:v4mv:vqmin=3:lumi_mask=0.07:dark_mask=0.2:"+\
"mpeg_quant:scplx_mask=0.1:tcplx_mask=0.1:naq -mf type=png:fps="+str(fps)+" -nosound -o "+movieName.replace('__tmp__','')+".mpg mf://"+movieName+"* > mencoder.log 2>&1"
os.system(ss)
#for png in pngs: os.system('rm '+png)
return 0
#!/usr/bin/env python
# -*- coding: utf-8 -*-
from losoto.lib_operations import *
from losoto._logging import logger as logging
logging.debug('Loading FLAGSTATION module.')
def _run_parser(soltab, parser, step):
mode = parser.getstr(step, 'mode') # no default
maxFlaggedFraction = parser.getfloat(step, 'maxFlaggedFraction', 0.5)
nSigma = parser.getfloat(step, 'nSigma', 5.0)
maxStddev = parser.getfloat(step, 'maxStddev', -1.0)
ampRange = parser.getarrayfloat(step, 'ampRange', [0., 0.])
telescope = parser.getstr(step, 'telescope', 'lofar')
skipInternational = parser.getbool(step, 'skipInternational', False)
refAnt = parser.getstr(step, 'refAnt', '')
soltabExport = parser.getstr(step, 'soltabExport', '')
ncpu = parser.getint('_global', 'ncpu', 0)
parser.checkSpelling(step, soltab, [
'mode', 'maxFlaggedFraction', 'nSigma', 'maxStddev', 'ampRange',
'telescope', 'skipInternational', 'refAnt', 'soltabExport'
])
return run(soltab, mode, maxFlaggedFraction, nSigma, maxStddev, ampRange,
telescope, skipInternational, refAnt, soltabExport, ncpu)
def _flag_resid(vals, weights, soltype, nSigma, maxFlaggedFraction, maxStddev,
ants, s, outQueue):
def doFit(
phases,
mask,
freqs,
stations,
station_positions,
axes,
refstIdx='superterp',
flagBadChannels=True,
flagcut=1.5,
chi2cut=30000.,
removePhaseWraps=True,
combine_pol=False,
fit3rdorder=False,
circular=False,
initSol=[],
initoffsets=[],
n_proc=10
):
# make sure order of axes is as expected
stidx = axes.index('ant')
freqidx = axes.index('freq')
timeidx = axes.index('time')
try:
polidx = axes.index('pol')
data = ma.array(phases, mask=mask).transpose((timeidx, freqidx, stidx, polidx))
npol = data.shape[3]
except:
data = ma.array(phases, mask=mask).transpose((timeidx, freqidx, stidx))
data = data.reshape(data.shape+(1,))
npol = 1
nT = data.shape[0]
nF = data.shape[1]
nSt = data.shape[2]
if npol == 4:
data = data[:, :, :, (0, 3)]
npol = 2
if refstIdx == 'superterp':
superterpstations = [i for i in stations if i[:5] in [
'CS002',
'CS003',
'CS004',
'CS005',
'CS006',
'CS007'
]]
refstIdx = [i for (i, j) in enumerate(stations) if j in superterpstations]
if len(refstIdx)==0:
refstIdx=0
if not hasattr(refstIdx, '__len__'):
refstIdx = [refstIdx]
# get phases from reference stations
refdata = np.ma.sum(np.cos(data[:, :, refstIdx, :]) + 1.j * np.sin(data[:, :, refstIdx, :]), axis=2)
refdata = np.ma.arctan2(np.imag(refdata), np.real(refdata))
# unwrap around mean
mymean = np.ma.average(refdata, axis=0)
refdata = np.ma.remainder(refdata - mymean[np.newaxis] + np.pi, 2 * np.pi) + mymean[np.newaxis] - np.pi
data -= refdata[:, :, np.newaxis]
# flag bad channels - test if really nedded if data already flagged
indices = np.arange(nF)
if flagBadChannels:
mymask=np.zeros((nF), dtype=np.bool)
for nr_iter in range(2):
rms = np.ma.std(np.ma.std(refdata, axis=0), axis=1)
freqselect = rms < flagcut * np.average(rms)
logging.debug('Iter %d: flagging %d channels' % (nr_iter, np.sum(np.logical_not(np.array(freqselect)))))
#freqs = freqs[freqselect]
#data = data[:, freqselect]
mymask=np.ma.logical_or(mymask,~freqselect)
#refdata = refdata[:, freqselect]
refdata.mask=np.ma.logical_or(refdata.mask,mymask[np.newaxis,:,np.newaxis])
logging.debug('Flagging: ' + str(indices[np.logical_not(freqselect)]))
#indices = indices[freqselect]
mask=data.mask
data.mask=np.logical_or(mask,mymask[np.newaxis,:,np.newaxis,np.newaxis])
# Remove completely flagged stations
selectstations = [st for stationIndices, st in enumerate(stations) if data[:, :, stationIndices].mask.all() != True]
logging.debug('%d selected stations: ' % len(selectstations) + str(selectstations))
stationIndices = np.array([idxst in selectstations for idxst in stations])
data = data[:, :, stationIndices]
stations_old = stations[:] # keep a copy for putting flagged stations back at the end
stations = stations[stationIndices]
station_positions = station_positions[stationIndices]
RSstations = [i for (i, j) in enumerate(stations) if 'RS' in j]
CSstations = [i for (i, j) in enumerate(stations) if 'CS' in j]
otherstations = [i for (i, j) in enumerate(stations) if not 'RS' in j and not 'CS' in j]
logging.debug('Station indices: ' + str(stationIndices) + ' RS ' + str(RSstations))
nSt = data.shape[2]
# combine polarizationsif requested - needed in HBA
if combine_pol:
if npol == 2 and not circular:
cdata = np.ma.cos(data) + 1.j * np.ma.sin(data)
data = np.ma.sum(cdata, axis=3).reshape((nT, nF, nSt, 1))
data = np.ma.arctan2(np.imag(data), np.real(data)) # np.angle doesnot yet return masked array!!
npol = 1
if circular:
data=np.ma.sum(data, axis=3).reshape((nT, nF, nSt, 1))
npol=1
# guess clock, remove from data
# not in LBA because TEC dominant
if not 'LBA' in stations[0] and len(initSol) < 1:
initclock = getInitClock(data[int(nT / 2):int(nT / 2 + 100)][:, :, RSstations + otherstations], freqs) # only on a few timestamps
logging.debug('Initial clocks: ' + str(initclock[1]))
# init CS clocks to 0
# logging.debug("data before init clock" + str(data[nT/2,:,-1]))
data[:, :, RSstations + otherstations] = data[:, :, RSstations + otherstations] - freqs[np.newaxis, :, np.newaxis, np.newaxis] * initclock[1][np.newaxis, np.newaxis, :] * 1e-9 * 2 * np.pi
# logging.debug("clock correction" + str(np.remainder(freqs*initclock[1][-1]*-1e-9*2*np.pi+np.pi,2*np.pi)-np.pi))
# logging.debug("data after init clock" + str(np.remainder(data[nT/2,:,-1]+np.pi,2*np.pi)-np.pi))
offset = np.zeros((nSt, npol), dtype=np.float32)
if len(initoffsets)>0: # Check if initoffsets is not empty
offset = initoffsets
data[:, :, :, :] += offset[:][np.newaxis, np.newaxis]
# initialize arrays
clock = np.zeros((nT, nSt, npol), dtype=np.float32)
tec = np.zeros((nT, nSt, npol), dtype=np.float32)
if fit3rdorder:
tec3rd = np.zeros((nT, nSt, npol), dtype=np.float32)
# better not to use fitoffset
tecarray = np.zeros((nT, nSt), dtype=np.float32)
clockarray = np.zeros((nT, nSt), dtype=np.float32)
residualarray = np.zeros((nT, nF, nSt), dtype=np.float32)
if fit3rdorder:
tec3rdarray= np.zeros((nT, nSt), dtype=np.float32)
for pol in range(npol):
# get a good guesss without offset
# logging.debug("sending masked data "+str(data[:,:,:,pol].count()))
initialchi2cut = chi2cut # user defined
if removePhaseWraps:
initialchi2cut = 30000. # this number is quite arbitrary
if fit3rdorder:
poolargs=[]
for ist in range(nSt):
poolargs.append((np.ma.copy(data[:, :, ist, pol]),
freqs,
stations[ist],
[],
True,
initialchi2cut,
True,
circular and combine_pol
))
p=Pool(processes=n_proc)
result=p.map(merge_ct_args,poolargs)
for ist,tc in enumerate(result):
tecarray[:, ist], clockarray[:, ist],residualarray[:,:,ist],tec3rdarray[:,ist] = tc
else:
poolargs=[]
for ist in range(nSt):
poolargs.append((np.ma.copy(data[:, :, ist, pol]),
freqs,
stations[ist],
[],
True,
initialchi2cut,
False,
circular and combine_pol
))
p=Pool(processes=n_proc)
result=p.map(merge_ct_args,poolargs)
for ist,tc in enumerate(result):
tecarray[:, ist], clockarray[:, ist],residualarray[:,:,ist] = tc
if removePhaseWraps:
# correctfrist times only,try to make init correct ?
#corrects wraps based on spatial correlation (averaged in time), only works for long time observations, not testted for LBA
(offset[:, pol], wraps, steps) = correctWraps(tecarray, residualarray, freqs, station_positions)
else:
#always correct for wraps based on average residuals
wraps, steps = correctWrapsFromResiduals(residualarray, tecarray<-5,freqs)
#maximum allowed 2pi phase wrap is +-1
#wraps = np.minimum(wraps,2)
#wraps = np.maximum(wraps,-2)
logging.debug('Residual iter 1, pol %d: ' % pol + str(residualarray[0, 0]))
logging.debug('TEC iter 1, pol %d: ' % pol + str(tecarray[0]))
logging.debug('Clock iter 1, pol %d: ' % pol + str(clockarray[0]))
logging.debug('Wraps: ' + str(wraps))
logging.debug('Offsets: ' + str(offset[:, pol]))
# remove completely initialoffset?
if len(initoffsets)>0: # Check if initoffsets is not empty
offset[:, pol] -= initoffsets[:, pol]
data[:, :, :, pol] += offset[:, pol][np.newaxis, np.newaxis]
# remove fitoffset
if removePhaseWraps:
initsol = np.zeros((nSt, 2), dtype=np.float32)
initsol[:, 0] = get_first_good(tecarray[:, :]) + wraps * steps[0]
initsol[:, 1] = get_first_good(clockarray[:, :]) + wraps * steps[1]
logging.debug('Initsol TEC, pol %d: ' % pol + str(initsol[:, 0]))
logging.debug('Initsol clock, pol %d: ' % pol + str(initsol[:, 1]))
# is it needed to redo the fitting? this is the time bottleneck
if fit3rdorder:
poolargs=[]
for ist in range(nSt):
poolargs.append((np.ma.copy(data[:, :, ist, pol]),
freqs,
stations[ist],
initsol[ist],
False,
chi2cut,
True,
circular and combine_pol
))
p=Pool(processes=n_proc)
result=p.map(merge_ct_args,poolargs)
for ist,tc in enumerate(result):
tec[:, ist, pol], clock[:, ist, pol],tec3rd[:,ist,pol] = tc
else:
poolargs=[]
for ist in range(nSt):
poolargs.append((np.ma.copy(data[:, :, ist, pol]),
freqs,
stations[ist],
initsol[ist],
False,
chi2cut,
False,
circular and combine_pol
))
p=Pool(processes=n_proc)
result=p.map(merge_ct_args,poolargs)
for ist,tc in enumerate(result):
tec[:, ist, pol], clock[:, ist, pol] = tc
else:
tec[:, :, pol] = tecarray[:, :]+ wraps * steps[0]
clock[:, :, pol] = clockarray[:, :]+ wraps * steps[1]
if fit3rdorder:
tec3rd[:, :, pol] = tec3rdarray[:, :]+ wraps * steps[2]
logging.debug('TEC iter 2, pol %d: ' % pol + str(tec[0, :, pol]))
logging.debug('Clock iter 2, pol %d: ' % pol + str(clock[0, :, pol]))
if not 'LBA' in stations[0] and len(initSol) < 1:
clock[:, RSstations + otherstations] += initclock[1][np.newaxis, :, :]
if combine_pol and circular:
clock/=2;tec/=2;offset/=2
# put flagged stations back
for idx, wasUsed in enumerate(stationIndices):
if not wasUsed:
logging.debug('Adding completely flagged station '+str(idx))
clock = np.insert(clock,idx,0,axis=1) # [time:ant:pol]
tec = np.insert(tec,idx,-5,axis=1) # [time:ant:pol]
offset = np.insert(offset,idx,0,axis=0) # [ant:pol]
if fit3rdorder: tec3rd = np.insert(tec3rd,idx,0,axis=1) # [time:ant:pol]
# station = station_old
if fit3rdorder:
return (clock, tec, offset, tec3rd)
else:
return (clock, tec, offset)
#!/usr/bin/env python
# -*- coding: utf-8 -*-
from __future__ import print_function
from losoto.lib_operations import *
from losoto._logging import logger as logging
logging.debug('Loading BANDPASSTEC module.')
def _run_parser(soltab, parser, step):
soltabOutTec = parser.getstr(step, 'soltabOutTEC', 'tec000')
soltabOutBP = parser.getstr(step, 'soltabOutBP', 'phase000')
refAnt = parser.getstr(step, 'refAnt', '')
parser.checkSpelling(
step, soltab, ['soltabOutTEC', 'soltabOutBP', 'refAnt', 'maxResidual'])
return run(soltab, soltabOutTEC, soltabOutBP, refAnt)
def run(soltab, soltabOutTEC='tec000', soltabOutBP='phase000', refAnt=''):
"""
Isolate BP from TEC in a calibrator (uGMRT data)
Parameters
----------
soltabOutTEC : str, optional
output TEC table name (same solset), by deault "tec000".
soltabOutBP : str, optional
output bandpass table name (same solset), by deault "phase000".
def run(soltab,
soltabOut='tec000',
refAnt='',
maxResidualFlag=2.5,
maxResidualProp=1.):
"""
Bruteforce TEC extraction from phase solutions.
Parameters
----------
soltabOut : str, optional
output table name (same solset), by deault "tec".
refAnt : str, optional
Reference antenna, by default the first.
maxResidualFlag : float, optional
Max average residual in radians before flagging datapoint, by default 2.5 If 0: no check.
maxResidualProp : float, optional
Max average residual in radians before stop propagating solutions, by default 1. If 0: no check.
"""
import numpy as np
import scipy.optimize
from losoto.lib_unwrap import unwrap_2d
def mod(d):
return np.mod(d + np.pi, 2. * np.pi) - np.pi
drealbrute = lambda d, freq, y: np.sum(
np.abs(mod(-8.44797245e9 * d / freq) - y)) # bruteforce
dreal = lambda d, freq, y: mod(-8.44797245e9 * d[0] / freq) - y
#dreal2 = lambda d, freq, y: mod(-8.44797245e9*d[0]/freq + d[1]) - y
#dcomplex = lambda d, freq, y: np.sum( ( np.cos(-8.44797245e9*d/freq) - np.cos(y) )**2 ) + np.sum( ( np.sin(-8.44797245e9*d/freq) - np.sin(y) )**2 )
#def dcomplex( d, freq, y, y_pre, y_post):
# return np.sum( ( np.absolute( np.exp(-1j*8.44797245e9*d/freq) - np.exp(1j*y) ) )**2 ) + \
# .5*np.sum( ( np.absolute( np.exp(-1j*8.44797245e9*d/freq) - np.exp(1j*y_pre) ) )**2 ) + \
# .5*np.sum( ( np.absolute( np.exp(-1j*8.44797245e9*d/freq) - np.exp(1j*y_post) ) )**2 )
#dcomplex2 = lambda d, freq, y: abs(np.cos(-8.44797245e9*d[0]/freq + d[1]) - np.cos(y)) + abs(np.sin(-8.44797245e9*d[0]/freq + d[1]) - np.sin(y))
logging.info("Find TEC for soltab: " + soltab.name)
# input check
solType = soltab.getType()
if solType != 'phase':
logging.warning("Soltab type of " + soltab._v_name + " is of type " +
solType + ", should be phase. Ignoring.")
return 1
ants = soltab.getAxisValues('ant')
if refAnt != '' and refAnt != 'closest' and not refAnt in soltab.getAxisValues(
'ant', ignoreSelection=True):
logging.error('Reference antenna ' + refAnt + ' not found. Using: ' +
ants[1])
refAnt = ants[0]
if refAnt == '': refAnt = ants[0]
# times and ants needs to be complete or selection is much slower
times = soltab.getAxisValues('time')
# create new table
solset = soltab.getSolset()
soltabout = solset.makeSoltab(soltype = 'tec', soltabName = soltabOut, axesNames=['ant','time'], \
axesVals=[soltab.getAxisValues(axisName) for axisName in ['ant','time']], \
vals=np.zeros(shape=(soltab.getAxisLen('ant'),soltab.getAxisLen('time'))), \
weights=np.ones(shape=(soltab.getAxisLen('ant'),soltab.getAxisLen('time'))) )
soltabout.addHistory('Created by TEC operation from %s.' % soltab.name)
for vals, weights, coord, selection in soltab.getValuesIter(
returnAxes=['freq', 'time'], weight=True, reference=refAnt):
if len(coord['freq']) < 10:
logging.error(
'Delay estimation needs at least 10 frequency channels, preferably distributed over a wide range.'
)
return 1
# reorder axes
vals = reorderAxes(vals, soltab.getAxesNames(), ['freq', 'time'])
weights = reorderAxes(weights, soltab.getAxesNames(), ['freq', 'time'])
fitd = np.zeros(len(times))
fitweights = np.ones(len(times)) # all unflagged to start
#fitdguess = 0.01 # good guess
ranges = (-0.5, 0.5)
Ns = 1000
if not coord['ant'] == refAnt:
if (weights == 0.).all() == True:
logging.warning('Skipping flagged antenna: ' + coord['ant'])
fitweights[:] = 0
else:
# unwrap 2d timexfreq
#flags = np.array((weights == 0), dtype=bool)
#vals = unwrap_2d(vals, flags, coord['freq'], coord['time'])
for t, time in enumerate(times):
# apply flags
idx = (weights[:, t] != 0.)
freq = np.copy(coord['freq'])[idx]
if t == 0: phaseComb_pre = vals[idx, 0]
else: phaseComb_pre = vals[idx, t - 1]
if t == len(times) - 1: phaseComb_post = vals[idx, -1]
else: phaseComb_post = vals[idx, t + 1]
phaseComb = vals[idx, t]
if len(freq) < 10:
fitweights[t] = 0
logging.warning(
'No valid data found for delay fitting for antenna: '
+ coord['ant'] + ' at timestamp ' + str(t))
continue
# if more than 1/4 of chans are flagged
if (len(idx) - len(freq)) / float(len(idx)) > 1 / 3.:
logging.debug(
'High number of filtered out data points for the timeslot %i: %i/%i'
% (t, len(idx) - len(freq), len(idx)))
# least square 2
#fitresultd2, success = scipy.optimize.leastsq(dreal2, [fitdguess,0.], args=(freq, phaseComb))
#numjumps = np.around(fitresultd2[1]/(2*np.pi))
#print 'best jumps:', numjumps
#phaseComb -= numjumps * 2*np.pi
#fitresultd, success = scipy.optimize.leastsq(dreal, [fitresultd2[0]], args=(freq, phaseComb))
# least square 1
#fitresultd, success = scipy.optimize.leastsq(dreal, [fitdguess], args=(freq, phaseComb))
# hopper
#fitresultd = scipy.optimize.basinhopping(dreal, [fitdguess], T=1, minimizer_kwargs={'args':(freq, phaseComb)})
#fitresultd = [fitresultd.x]
#best_residual = np.nanmean(np.abs( mod(-8.44797245e9*fitresultd[0]/freq) - phaseComb ) )
#best_residual = np.inf
#for jump in [-2,-1,0,1,2]:
# fitresultd, success = scipy.optimize.leastsq(dreal, [fitdguess], args=(freq, phaseComb - jump * 2*np.pi))
# print fitresultd
# # fractional residual
# residual = np.nanmean(np.abs( (-8.44797245e9*fitresultd[0]/freq) - phaseComb - jump * 2*np.pi ) )
# if residual < best_residual:
# best_residual = residual
# fitd[t] = fitresultd[0]
# best_jump = jump
# brute force
fitresultd = scipy.optimize.brute(drealbrute,
ranges=(ranges, ),
Ns=Ns,
args=(freq, phaseComb))
fitresultd, success = scipy.optimize.leastsq(
dreal, fitresultd, args=(freq, phaseComb))
best_residual = np.nanmean(
np.abs(
mod(-8.44797245e9 * fitresultd[0] / freq) -
phaseComb))
fitd[t] = fitresultd[0]
if maxResidualFlag == 0 or best_residual < maxResidualFlag:
fitweights[t] = 1
if maxResidualProp == 0 or best_residual < maxResidualProp:
ranges = (fitresultd[0] - 0.05,
fitresultd[0] + 0.05)
Ns = 100
else:
ranges = (-0.5, 0.5)
Ns = 1000
else:
# high residual, flag and reset initial guess
logging.warning('Bad solution for ant: ' +
coord['ant'] + ' (time: ' + str(t) +
', resdiual: ' + str(best_residual) +
').')
fitweights[t] = 0
ranges = (-0.5, 0.5)
Ns = 1000
# Debug plot
doplot = False
if doplot and (coord['ant'] == 'RS509LBA' or coord['ant']
== 'RS210LBA') and t % 50 == 0:
print("Plotting")
if not 'matplotlib' in sys.modules:
import matplotlib as mpl
mpl.rc('figure.subplot',
left=0.05,
bottom=0.05,
right=0.95,
top=0.95,
wspace=0.22,
hspace=0.22)
mpl.use("Agg")
import matplotlib.pyplot as plt
fig = plt.figure()
fig.subplots_adjust(wspace=0)
ax = fig.add_subplot(111)
# plot rm fit
plotd = lambda d, freq: -8.44797245e9 * d / freq
ax.plot(freq,
plotd(fitresultd[0], freq[:]),
"-",
color='purple')
ax.plot(freq,
mod(plotd(fitresultd[0], freq[:])),
":",
color='purple')
#ax.plot(freq, vals[idx,t], '.b' )
#ax.plot(freq, phaseComb + numjumps * 2*np.pi, 'x', color='purple' )
ax.plot(freq, phaseComb, 'o', color='purple')
residual = mod(plotd(fitd[t], freq[:]) - phaseComb)
ax.plot(freq, residual, '.', color='orange')
ax.set_xlabel('freq')
ax.set_ylabel('phase')
#ax.set_ylim(ymin=-np.pi, ymax=np.pi)
logging.warning('Save pic: ' + str(t) + '_' +
coord['ant'] + '.png')
plt.savefig(str(t) + '_' + coord['ant'] + '.png',
bbox_inches='tight')
del fig
logging.info('%s: average tec: %f TECU' %
(coord['ant'], np.mean(2 * fitd)))
# reorder axes back to the original order, needed for setValues
soltabout.setSelection(ant=coord['ant'])
soltabout.setValues(fitd)
soltabout.setValues(fitweights, weight=True)
return 0
#!/usr/bin/env python
# -*- coding: utf-8 -*-
from losoto.lib_operations import *
from losoto._logging import logger as logging
logging.debug('Loading POLALIGN module.')
def _run_parser(soltab, parser, step):
soltabOut = parser.getstr( step, 'soltabOut', 'phasediff' )
maxResidual = parser.getfloat( step, 'maxResidual', 1. )
fitOffset = parser.getbool( step, 'fitOffset', False )
average = parser.getbool( step, 'average', False )
replace = parser.getbool( step, 'replace', False )
minFreq = parser.getfloat( step, 'minFreq', 0 )
refAnt = parser.getstr( step, 'refAnt', '' )
parser.checkSpelling( step, soltab, ['soltabOut', 'maxResidual', 'fitOffset', 'average', 'replace', 'minFreq', 'refAnt'])
return run(soltab, soltabOut, maxResidual, fitOffset, average, replace, minFreq, refAnt)
def run( soltab, soltabOut='phasediff', maxResidual=1., fitOffset=False, average=False, replace=False, minFreq=0, refAnt='' ):
"""
Estimate polarization misalignment as delay.
Parameters
----------
soltabOut : str, optional
output table name (same solset), by deault "phasediff".
maxResidual : float, optional
