def run(step, parset, H):
from h5parm import solWriter
soltabs = getParSoltabs(step, parset, H)
ants = getParAxis(step, parset, H, 'ant')
pols = getParAxis(step, parset, H, 'pol')
dirs = getParAxis(step, parset, H, 'dir')
setWeight = parset.getBool('.'.join(["LoSoTo.Steps", step, "Weight"]),
False)
for soltab in openSoltabs(H, soltabs):
logging.info("Resetting soltab: " + soltab._v_name)
t = solWriter(soltab)
t.setSelection(ant=ants, pol=pols, dir=dirs)
solType = t.getType()
if setWeight:
t.setValues(1., weight=setWeight)
elif solType == 'amplitude':
t.setValues(1.)
else:
t.setValues(0.)
t.addHistory('RESET')
return 0
def run(step, parset, H):
"""
Copy values from a table to another (of the same kind)
If tables have different sampling, then resample the values
"""
import numpy as np
from h5parm import solFetcher, solWriter
inTable = parset.getString('.'.join(["LoSoTo.Steps", step, "InTable"]), '')
outTable = parset.getString('.'.join(["LoSoTo.Steps", step, "OutTable"]),
'')
tr = solFetcher(inTable)
inVals, inCoord = tr.getValues()
tw = solWriter(outTable)
outVals, outCoord = tw.getValues()
shapeDiff = tuple(np.array(outVals.shape) / np.array(inVals.shape))
inValsNew = np.kron(inVals, np.ones(shapeDiff))
if inValsNew.shape != outVals.shape:
logging.error(
"Incompatible shapes for copying solutions. The outTable axes can be a multiple of the inTable axis dimension."
)
# writing on the output table
tw.setValues(inValsNew)
tw.addHistory('COPY (from table %s)' % (inTable))
return 0
def run( step, parset, H ):
"""
Copy values from a table to another (of the same kind)
If tables have different sampling, then resample the values
"""
import numpy as np
from h5parm import solFetcher, solWriter
inTable = parset.getString('.'.join(["LoSoTo.Steps", step, "InTable"]), '' )
outTable = parset.getString('.'.join(["LoSoTo.Steps", step, "OutTable"]), '' )
tr = solFetcher(inTable)
inVals, inCoord = tr.getValues()
tw = solWriter(outTable)
outVals, outCoord = tw.getValues()
shapeDiff = tuple(np.array(outVals.shape)/np.array(inVals.shape))
inValsNew = np.kron(inVals, np.ones(shapeDiff))
if inValsNew.shape != outVals.shape:
logging.error("Incompatible shapes for copying solutions. The outTable axes can be a multiple of the inTable axis dimension.")
# writing on the output table
tw.setValues(inValsNew)
tw.addHistory('COPY (from table %s)' % (inTable))
return 0
def run( step, parset, H ):
import scipy.ndimage.filters
from h5parm import solFetcher, solWriter
soltabs = getParSoltabs( step, parset, H )
ants = getParAxis( step, parset, H, 'ant' )
pols = getParAxis( step, parset, H, 'pol' )
dirs = getParAxis( step, parset, H, 'dir' )
axesToSmooth = parset.getStringVector('.'.join(["LoSoTo.Steps", step, "Axes"]), [] )
FWHM = parset.getIntVector('.'.join(["LoSoTo.Steps", step, "FWHM"]), [] )
if len(axesToSmooth) != len(FWHM):
logging.error("Axes and FWHM lenghts must be equal.")
return 1
for soltab in openSoltabs( H, soltabs ):
sf = solFetcher(soltab)
sw = solWriter(soltab)
logging.info("Smoothing soltab: "+soltab._v_name)
sf.setSelection(ant=ants, pol=pols, dir=dirs)
for i, axis in enumerate(axesToSmooth[:]):
if axis not in sf.getAxesNames():
del axesToSmooth[i]
del FWHM[i]
logging.warning('Axis \"'+axis+'\" not found. Ignoring.')
for vals, coord in sf.getValuesIter(returnAxes=axesToSmooth):
valsnew = scipy.ndimage.filters.median_filter(vals, FWHM)
# TODO: implement flag control
# find the local mean (without any nans)
# valmean = np.ma.mean(np.ma.masked_array(vals, np.isnan(vals)), axis=0)
# print 'mean value: ' + str(valmean)
# replace any nans with median
# valbool = np.isnan(vals)
# nanindex = np.where(valbool)
# if valbool.any():
# logging.warning('Found NaNs in solutions. Replacing with local mean.')
# check if valmean is iterable
# if valmean.shape is tuple():
# np.putmask(vals, valbool, valmean)
# else:
# for x,mval in enumerate(valmean):
# np.putmask(vals[:,x], valbool[:,x], mval)
# writing back the solutions (selection on all the coord axis)
# this can be properly broacasted
coord = removeKeys(coord, axesToSmooth)
sw.setSelection(**coord)
sw.setValues(valsnew)
sw.addHistory('SMOOTH (over %s with box size = %s)' % (axesToSmooth, FWHM))
return 0
def run( step, parset, H ):
import scipy.ndimage.filters
from h5parm import solFetcher, solWriter
soltabs = getParSoltabs( step, parset, H )
ants = getParAxis( step, parset, H, 'ant' )
pols = getParAxis( step, parset, H, 'pol' )
dirs = getParAxis( step, parset, H, 'dir' )
axesToSmooth = parset.getStringVector('.'.join(["LoSoTo.Steps", step, "Axes"]), [] )
FWHM = parset.getIntVector('.'.join(["LoSoTo.Steps", step, "FWHM"]), [] )
if len(axesToSmooth) != len(FWHM):
logging.error("Axes and FWHM lenghts must be equal.")
return 1
for soltab in openSoltabs( H, soltabs ):
sf = solFetcher(soltab)
sw = solWriter(soltab)
logging.info("Smoothing soltab: "+soltab._v_name)
sf.setSelection(ant=ants, pol=pols, dir=dirs)
for i, axis in enumerate(axesToSmooth[:]):
if axis not in sf.getAxesNames():
del axesToSmooth[i]
del FWHM[i]
logging.warning('Axis \"'+axis+'\" not found. Ignoring.')
for vals, coord in sf.getValuesIter(returnAxes=axesToSmooth):
valsnew = scipy.ndimage.filters.median_filter(vals, FWHM)
# Assume 1D filtering
# window=FWHM[0]/2
# valsnew=np.zeros(len(vals))
# for i in xrange(0,len(vals)):
# min=i-window;
# if (min<0):
# min=0;
# max=i+window;
# if (max>=len(vals)):
# max=len(vals)-1
# valsnew[i]=np.median(vals[min:max])
coord = removeKeys(coord, axesToSmooth)
sw.setSelection(**coord)
sw.setValues(valsnew)
sw.addHistory('SMOOTH (over %s with box size = %s)' % (axesToSmooth, FWHM))
return 0
def run(step, parset, H):
"""
Normalize the solutions to a given value
"""
import numpy as np
from h5parm import solFetcher, solWriter
solsets = getParSolsets(step, parset, H)
soltabs = getParSoltabs(step, parset, H)
solTypes = getParSolTypes(step, parset, H)
ants = getParAxis(step, parset, H, 'ant')
pols = getParAxis(step, parset, H, 'pol')
dirs = getParAxis(step, parset, H, 'dir')
normVal = parset.getFloat('.'.join(["LoSoTo.Steps", step, "NormVal"]), 1.)
normAxis = parset.getString('.'.join(["LoSoTo.Steps", step, "NormAxis"]),
'time')
for soltab in openSoltabs(H, soltabs):
logging.info("--> Working on soltab: " + soltab._v_name)
tr = solFetcher(soltab)
tw = solWriter(soltab)
axesNames = tr.getAxesNames()
if normAxis not in axesNames:
logging.error('Normalization axis ' + normAxis + ' not found.')
return 1
tr.setSelection(ant=ants, pol=pols, dir=dirs)
for vals, coord in tr.getValuesIter(returnAxes=normAxis):
# construct grid
coordSel = removeKeys(coord, normAxis)
logging.debug("Working on coords:" + str(coordSel))
# rescale solutions
valsMean = np.mean(vals)
valsNew = normVal * vals / valsMean
logging.debug("Rescaling by: " + str(normVal / valsMean))
# writing back the solutions
tw.setSelection(**coordSel)
tw.setValues(valsNew)
tw.addHistory('NORM (on axis %s)' % (normAxis))
return 0
def run( step, parset, H ):
"""
Normalize the solutions to a given value
"""
import numpy as np
from h5parm import solFetcher, solWriter
solsets = getParSolsets( step, parset, H )
soltabs = getParSoltabs( step, parset, H )
solTypes = getParSolTypes( step, parset, H )
ants = getParAxis( step, parset, H, 'ant' )
pols = getParAxis( step, parset, H, 'pol' )
dirs = getParAxis( step, parset, H, 'dir' )
normVal = parset.getFloat('.'.join(["LoSoTo.Steps", step, "NormVal"]), 1. )
normAxis = parset.getString('.'.join(["LoSoTo.Steps", step, "NormAxis"]), 'time' )
for soltab in openSoltabs( H, soltabs ):
logging.info("--> Working on soltab: "+soltab._v_name)
tr = solFetcher(soltab)
tw = solWriter(soltab)
axesNames = tr.getAxesNames()
if normAxis not in axesNames:
logging.error('Normalization axis '+normAxis+' not found.')
return 1
tr.setSelection(ant=ants, pol=pols, dir=dirs)
for vals, coord in tr.getValuesIter(returnAxes=normAxis):
# construct grid
coordSel = removeKeys(coord, normAxis)
logging.debug("Working on coords:"+str(coordSel))
# rescale solutions
valsMean = np.mean(vals)
valsNew = normVal*vals/valsMean
logging.debug("Rescaling by: "+str(normVal/valsMean))
# writing back the solutions
tw.setSelection(**coordSel)
tw.setValues(valsNew)
tw.addHistory('NORM (on axis %s)' % (normAxis))
return 0
def run(step, parset, H):
import numpy as np
from h5parm import solFetcher, solWriter
soltabs = getParSoltabs(step, parset, H)
ants = getParAxis(step, parset, H, 'ant')
pols = getParAxis(step, parset, H, 'pol')
dirs = getParAxis(step, parset, H, 'dir')
# No need to specify an axis, just use time
axesToAbs = ['time']
for soltab in openSoltabs(H, soltabs):
sf = solFetcher(soltab)
sw = solWriter(soltab)
logging.info("Taking ABSolute value of soltab: " + soltab._v_name)
sf.setSelection(ant=ants, pol=pols, dir=dirs)
total = 0
count = 0
for vals, coord in sf.getValuesIter(returnAxes=axesToAbs):
total += len(vals)
count += np.count_nonzero(vals < 0)
valsnew = np.abs(vals)
# writing back the solutions
coord = removeKeys(coord, axesToAbs)
sw.setSelection(**coord)
sw.setValues(valsnew)
logging.info('Abs: %i points initially negative (%f %%)' %
(count, float(count) / total))
sw.addHistory('ABSolute value taken')
return 0
def run( step, parset, H ):
import numpy as np
from h5parm import solFetcher, solWriter
soltabs = getParSoltabs( step, parset, H )
ants = getParAxis( step, parset, H, 'ant' )
pols = getParAxis( step, parset, H, 'pol' )
dirs = getParAxis( step, parset, H, 'dir' )
# No need to specify an axis, just use time
axesToAbs = ['time']
for soltab in openSoltabs( H, soltabs ):
sf = solFetcher(soltab)
sw = solWriter(soltab)
logging.info("Taking ABSolute value of soltab: "+soltab._v_name)
sf.setSelection(ant=ants, pol=pols, dir=dirs)
total=0
count=0
for vals, coord in sf.getValuesIter(returnAxes=axesToAbs):
total+=len(vals)
count+=np.count_nonzero(vals<0)
valsnew=np.abs(vals)
# writing back the solutions
coord = removeKeys(coord, axesToAbs)
sw.setSelection(**coord)
sw.setValues(valsnew)
logging.info('Abs: %i points initially negative (%f %%)' % (count,float(count)/total))
sw.addHistory('ABSolute value taken')
return 0
def run( step, parset, H ):
"""
Interpolate the solutions from one table into a destination table
"""
import itertools
import scipy.interpolate
import numpy as np
from h5parm import solFetcher, solWriter
solsets = getParSolsets( step, parset, H )
soltabs = getParSoltabs( step, parset, H )
solTypes = getParSolTypes( step, parset, H )
ants = getParAxis( step, parset, H, 'ant' )
pols = getParAxis( step, parset, H, 'pol' )
dirs = getParAxis( step, parset, H, 'dir' )
calSoltab = parset.getString('.'.join(["LoSoTo.Steps", step, "CalSoltab"]), '' )
calDir = parset.getString('.'.join(["LoSoTo.Steps", step, "CalDir"]), '' )
interpAxes = parset.getStringVector('.'.join(["LoSoTo.Steps", step, "InterpAxes"]), ['time','freq'] )
interpMethod = parset.getString('.'.join(["LoSoTo.Steps", step, "InterpMethod"]), 'linear' )
medAxes = parset.getStringVector('.'.join(["LoSoTo.Steps", step, "MedAxes"]), [''] )
rescale = parset.getBool('.'.join(["LoSoTo.Steps", step, "Rescale"]), False )
if interpMethod not in ["nearest", "linear", "cubic"]:
logging.error('Interpolation method must be nearest, linear or cubic.')
return 1
if rescale and medAxes == []:
logging.error('A medAxis is needed for rescaling.')
return 1
# open calibration table
css, cst = calSoltab.split('/')
cr = solFetcher(H.getSoltab(css, cst))
cAxesNames = cr.getAxesNames()
for soltab in openSoltabs( H, soltabs ):
logging.info("--> Working on soltab: "+soltab._v_name)
tr = solFetcher(soltab)
tw = solWriter(soltab)
axesNames = tr.getAxesNames()
for i, interpAxis in enumerate(interpAxes[:]):
if interpAxis not in axesNames or interpAxis not in cAxesNames:
logging.error('Axis '+interpAxis+' not found. Ignoring.')
del interpAxes[i]
for i, medAxis in enumerate(medAxes[:]):
if medAxis not in axesNames or medAxis not in cAxesNames:
logging.error('Axis '+medAxis+' not found. Ignoring.')
del medAxes[i]
tr.setSelection(ant=ants, pol=pols, dir=dirs)
for vals, coord in tr.getValuesIter(returnAxes=interpAxes):
# construct grid
coordSel = removeKeys(coord, interpAxes)
logging.debug("Working on coords:"+str(coordSel))
# change dir if sepcified
if calDir != '':
coordSel['dir'] = calDir
cr.setSelection(**coordSel)
calValues, calCoord = cr.getValues()
# fill all medAxes with the median value
for medAxis in medAxes:
axis = cAxesNames.index(medAxis)
calValues = np.repeat( np.expand_dims( np.median( calValues, axis ), axis ), calValues.shape[axis], axis )
# create a list of values whose coords are calPoints
calValues = np.ndarray.flatten(calValues)
# create calibrator/target coordinates arrays
calPoints = []
targetPoints = []
for interpAxis in interpAxes:
calPoints.append(calCoord[interpAxis])
targetPoints.append(coord[interpAxis])
calPoints = np.array([x for x in itertools.product(*calPoints)])
targetPoints = np.array([x for x in itertools.product(*targetPoints)])
# interpolation
valsNew = scipy.interpolate.griddata(calPoints, calValues, targetPoints, interpMethod)
# fill values outside boudaries with "nearest" solutions
if interpMethod != 'nearest':
valsNewNearest = scipy.interpolate.griddata(calPoints, calValues, targetPoints, 'nearest')
valsNew[ np.where(valsNew == np.nan) ] = valsNewNearest [ np.where(valsNew == np.nan) ]
valsNew = valsNew.reshape(vals.shape)
if rescale:
# rescale solutions
for medAxis in medAxes:
axis = interpAxes.index(medAxis)
valsMed = np.repeat( np.expand_dims( np.median( vals, axis ), axis ), vals.shape[axis], axis )
valsNewMed = np.repeat( np.expand_dims( np.median( valsNew, axis ), axis ), valsNew.shape[axis], axis )
valsNew = vals*valsNewMed/valsMed
#print "Rescaling by: ", valsNewMed[:,0]/valsMed[:,0]
# writing back the solutions
coord = removeKeys(coord, interpAxes)
tw.setSelection(**coord)
tw.setValues(valsNew)
tw.addHistory('INTERP (from table %s)' % (calSoltab))
return 0
def run(step, parset, H):
"""
Fit phase solutions to obtain TEC values per direction.
Phase solutions are assumed to be stored in solsets of the H5parm file, one
solset per band per field. Only phase- or scalarphase-type solution tables
are used. If direction-independent solutions are found (in addition to the
direction-dependent ones), they are added, after averaging, to the
corresponding direction-dependent ones. Phase solutions are automatically
grouped by field and by band.
The derived TEC values are stored in the specified output soltab of type
'tec', with one TEC value per station per direction per solution interval.
The TEC values are derived using the ``lofar.expion.baselinefitting.fit()``
function to fit a TEC value to the phases. The term that is minimized
includes all baselines, so there is no preferred reference station, and the
residual is computed as the complex number 1.0 - exp(1i phasedifference),
which is zero when the phase difference is a multiple of 2pi.
The TEC solution table may be used to derive TEC screens using the
TECSCREEN operation.
"""
from h5parm import solFetcher, solWriter
import numpy as np
from pylab import find
import re
solsets = getParSolsets(step, parset, H)
ants = getParAxis(step, parset, H, 'ant')
pols = getParAxis(step, parset, H, 'pol')
dirs = getParAxis(step, parset, H, 'dir')
nband_min = int(
parset.getString('.'.join(["LoSoTo.Steps", step, "MinBands"]), '8'))
dist_cut_m = np.float(
parset.getString('.'.join(["LoSoTo.Steps", step, "DistCut"]), '2e3'))
nstations_max = int(
parset.getString('.'.join(["LoSoTo.Steps", step, "MaxStations"]),
'100'))
outSolset = parset.getString('.'.join(["LoSoTo.Steps", step, "OutSoltab"]),
'').split('/')[0]
outSoltab = parset.getString('.'.join(["LoSoTo.Steps", step, "OutSoltab"]),
'').split('/')[1]
# Collect solutions, etc. into arrays for fitting.
(phases0, phases1, flags, mask, station_names, station_positions,
source_names, source_positions, freqs, times, pointing,
soln_type) = collect_solutions(H, dirs=dirs, solsets=solsets)
if phases0 is None:
return 1
# Build list of stations to include
included_stations = []
if ants is not None:
for ant in ants:
included_stations += [
s for s in station_names if re.search(ant, s)
]
else:
included_stations = station_names
excluded_stations = [
s for s in station_names if s not in included_stations
]
# Select stations to use for first pass
if len(excluded_stations) > 0:
logging.info("Excluding stations: {0}".format(
np.sort(excluded_stations)))
mean_position = np.array([
np.median(station_positions[:, 0]),
np.median(station_positions[:, 1]),
np.median(station_positions[:, 2])
])
dist = np.sqrt(np.sum((station_positions - mean_position)**2, axis=1))
dist_sort_ind = np.argsort(dist)
station_selection1 = find(dist < dist_cut_m)
station_selection = np.array([
i for i in dist_sort_ind if i in station_selection1
and station_names[i] not in excluded_stations
])
if len(station_selection) > nstations_max:
station_selection = station_selection[:nstations_max]
logging.info("Using normal fitting (no iterative search) for {0} stations "
"within {1} km of the core:\n{2}".format(
len(station_selection), dist_cut_m / 1000.0,
station_names[station_selection]))
# Fit a TEC value to the phase solutions per source pair. No iterative search for the
# global minimum is done
if soln_type == 'scalarphase':
r, source_selection = fit_tec_per_source_pair(
phases0[:, station_selection, :, :],
flags[:, station_selection, :, :],
mask,
freqs,
propagate=True,
nband_min=nband_min)
if r is None:
return 1
else:
r0, source_selection = fit_tec_per_source_pair(
phases0[:, station_selection, :, :],
flags[:, station_selection, :, :],
mask,
freqs,
propagate=True,
nband_min=nband_min)
r1, source_selection = fit_tec_per_source_pair(
phases1[:, station_selection, :, :],
flags[:, station_selection, :, :],
mask,
freqs,
propagate=True,
nband_min=nband_min)
if r0 is None or r1 is None:
return 1
# take the mean of the two polarizations
r = (r0 + r1) / 2
# Add stations by searching iteratively for global minimum in solution space
station_selection, r = add_stations(station_selection,
phases0,
phases1,
flags,
mask,
station_names,
station_positions,
source_names,
source_selection,
times,
freqs,
r,
nband_min=nband_min,
soln_type=soln_type,
nstations_max=nstations_max,
excluded_stations=excluded_stations,
search_full_tec_range=False)
# Save TEC values to the output solset
solset = H.makeSolset(outSolset)
dirs_out = []
dirs_pos = []
for s in source_selection:
dirs_out.append(source_names[s])
dirs_pos.append(source_positions[s])
sourceTable = solset._f_get_child('source')
sourceTable.append(zip(*(dirs_out, dirs_pos)))
times_out = times
ants_out = []
ants_pos = []
for s in station_selection:
ants_out.append(station_names[s])
ants_pos.append(station_positions[s])
antennaTable = solset._f_get_child('antenna')
antennaTable.append(zip(*(ants_out, ants_pos)))
tf_st = H.makeSoltab(solset._v_name,
'tec',
outSoltab,
axesNames=['dir', 'time', 'ant'],
axesVals=[dirs_out, times, ants_out],
vals=r[source_selection, :, :],
weights=np.ones_like(r[source_selection, :, :]))
# Add history
sw = solWriter(tf_st)
sw.addHistory('CREATE (by TECFIT operation)')
return 0
def run(step, parset, H):
import numpy as np
from h5parm import solFetcher, solWriter
soltabs = getParSoltabs(step, parset, H)
ants = getParAxis(step, parset, H, 'ant')
pols = getParAxis(step, parset, H, 'pol')
dirs = getParAxis(step, parset, H, 'dir')
axesToClip = parset.getStringVector(
'.'.join(["LoSoTo.Steps", step, "Axes"]), [])
clipLevel = parset.getFloat('.'.join(["LoSoTo.Steps", step, "ClipLevel"]),
0.)
if len(axesToClip) < 1:
logging.error("Please specify axes to clip.")
return 1
if clipLevel == 0.:
logging.error(
"Please specify factor above/below median at which to clip.")
return 1
for soltab in openSoltabs(H, soltabs):
sf = solFetcher(soltab)
sw = solWriter(soltab)
logging.info("Clipping soltab: " + soltab._v_name)
sf.setSelection(ant=ants, pol=pols, dir=dirs)
# some checks
for i, axis in enumerate(axesToClip[:]):
if axis not in sf.getAxesNames():
del axesToClip[i]
logging.warning('Axis \"' + axis + '\" not found. Ignoring.')
before_count = 0
after_count = 0
total = 0
for vals, coord in sf.getValuesIter(returnAxes=axesToClip):
total += len(vals)
before_count += np.count_nonzero(np.isnan(vals))
# clipping
# first find the median and standard deviation
valmedian = np.median(vals)
clipvalue = valmedian * clipLevel
np.putmask(vals, vals > clipvalue, np.nan)
clipvalue = valmedian / clipLevel
np.putmask(vals, vals < clipvalue, np.nan)
after_count += np.count_nonzero(np.isnan(vals))
# writing back the solutions
coord = removeKeys(coord, axesToClip)
sw.setSelection(**coord)
sw.setValues(vals)
sw.addHistory('CLIP (over %s with %s sigma cut)' %
(axesToClip, clipLevel))
logging.info(
'Clip: %i points initially bad, %i after clipping (%f %%)' %
(before_count, after_count, float(after_count) / total))
return 0
def run( step, parset, H ):
import numpy as np
from h5parm import solFetcher, solWriter
soltabs = getParSoltabs( step, parset, H )
ants = getParAxis( step, parset, H, 'ant' )
pols = getParAxis( step, parset, H, 'pol' )
dirs = getParAxis( step, parset, H, 'dir' )
axesToMed = parset.getStringVector('.'.join(["LoSoTo.Steps", step, "Axes"]), [] )
clipLevel = parset.getFloat('.'.join(["LoSoTo.Steps", step, "SigLevel"]), 0. )
if len(axesToMed) < 1:
logging.error("Please specify axes.")
return 1
if clipLevel == 0.:
logging.error("Please specify significance level above which to clip.")
return 1
for soltab in openSoltabs( H, soltabs ):
sf = solFetcher(soltab)
sw = solWriter(soltab)
logging.info("Flagging soltab: "+soltab._v_name)
sf.setSelection(ant=ants, pol=pols, dir=dirs)
# some checks
if len(axesToMed) < 1:
logging.error("Please specify axes.")
return 1
if clipLevel == 0.:
logging.error("Please specify significance level above which to clip.")
return 1
for i, axis in enumerate(axesToMed[:]):
if axis not in sf.getAxesNames():
del axesToMed[i]
logging.warning('Axis \"'+axis+'\" not found. Ignoring.')
before_count=0
after_count=0
total=0
for vals, coord in sf.getValuesIter(returnAxes=axesToMed):
total+=len(vals)
before_count+=np.count_nonzero(np.isnan(vals))
# clipping
# first find the median and standard deviation
valmedian = np.median(vals)
clipvalue = valmedian * clipLevel
np.putmask(vals, vals > clipvalue, np.nan)
clipvalue = valmedian / clipLevel
np.putmask(vals, vals < clipvalue, np.nan)
after_count+=np.count_nonzero(np.isnan(vals))
# writing back the solutions
coord = removeKeys(coord, axesToMed)
sw.setSelection(**coord)
sw.setValues(vals)
sw.addHistory('FLAG (over %s with %s sigma cut)' % (axesToMed, clipLevel))
logging.info('Clip: %i points initially bad, %i after flagging (%f %%)' % (before_count,after_count,float(after_count)/total))
return 0
def run(step, parset, H):
"""
Fits a screen to TEC values derived by the TECFIT operation.
The TEC values are read from the specified tec soltab.
The results of the fit are stored in the specified tecscreen solution table.
These values are the screen TEC values per station per pierce point per
solution interval. The pierce point locations are stored in an auxiliary
array in the output solution table.
TEC screens can be plotted with the PLOT operation by setting PlotType =
TECScreen.
The H5parm_exporter.py tool can be used to export the screen to a parmdb
that BBS and the AWimager can use. Note, however, that the output screens
are not normalized properly (any normalization was lost due to the use of
source-to-source phase gradients in the TECFIT operation). Therefore, a
direction-independent calibration must be done after exporting the screens
to a parmdb file, with the following settings in the BBS solve step:
Model.Ionosphere.Enable = T
Model.Ionosphere.Type = EXPION
"""
import numpy as np
import re
from h5parm import solFetcher, solWriter
soltabs = getParSoltabs(step, parset, H)
outSoltabs = parset.getStringVector(
'.'.join(["LoSoTo.Steps", step, "OutSoltab"]), [])
height = np.array(
parset.getDoubleVector('.'.join(["LoSoTo.Steps", step, "Height"]),
[200e3]))
order = int(
parset.getString('.'.join(["LoSoTo.Steps", step, "Order"]), '15'))
# Load TEC values from TECFIT operation
indx = 0
for soltab in openSoltabs(H, soltabs):
if 'tec' not in soltab._v_title:
logging.warning(
'No TECFIT solution tables found for solution table '
'{0}'.format(soltabs[indx]))
continue
indx += 1
solset = soltabs[indx].split('/')[0]
logging.info('Using input solution table: {0}'.format(soltabs[indx]))
logging.info('Using output solution table: {0}'.format(
outSoltabs[indx]))
# Collect station and source names and positions and times, making sure
# that they are ordered correctly.
t = solFetcher(soltab)
r, axis_vals = t.getValues()
source_names = axis_vals['dir']
source_dict = H.getSou(solset)
source_positions = []
for source in source_names:
source_positions.append(source_dict[source])
station_names = axis_vals['ant']
station_dict = H.getAnt(solset)
station_positions = []
for station in station_names:
station_positions.append(station_dict[station])
times = axis_vals['time']
# Get sizes
N_sources = len(source_names)
N_times = len(times)
N_stations = len(station_names)
N_piercepoints = N_sources * N_stations
rr = np.reshape(r.transpose([0, 2, 1]), [N_piercepoints, N_times])
heights = set(np.linspace(height[0], height[-1], 5))
if len(heights) > 1:
logging.info('Trying range of heights: {0} m'.format(
list(heights)))
for i, height in enumerate(heights):
# Find pierce points and airmass values for given screen height
logging.info('Using height = {0} m and order = {1}'.format(
height, order))
pp, airmass = calculate_piercepoints(np.array(station_positions),
np.array(source_positions),
np.array(times), height)
# Fit a TEC screen
r_0 = 10e3
beta = 5.0 / 3.0
tec_screen, residual = fit_screen_to_tec(station_names,
source_names, pp, airmass,
rr, times, height, order,
r_0, beta)
total_resid = np.sum(np.abs(residual))
if i > 0:
if total_resid < best_resid:
tec_screen_best = tec_screen
pp_best = pp
height_best = height
best_resid = total_resid
else:
tec_screen_best = tec_screen
pp_best = pp
height_best = height
best_resid = total_resid
if len(heights) > 1:
logging.info(
'Total residual for fit: {0}\n'.format(total_resid))
# Use screen with lowest total residual
if len(heights) > 1:
tec_screen = tec_screen_best
pp = pp_best
height = height_best
logging.info(
'Using height (with lowest total residual) of {0} m'.format(
height))
# Write the results to the output solset
dirs_out = source_names
times_out = times
ants_out = station_names
# Make output tecscreen table
outSolset = outSoltabs[indx].split('/')[0]
outSoltab = outSoltabs[indx].split('/')[1]
if not outSolset in H.getSolsets().keys():
solsetTEC = H.makeSolset(outSolset)
dirs_pos = source_positions
sourceTable = solsetTEC._f_get_child('source')
sourceTable.append(zip(*(dirs_out, dirs_pos)))
ants_pos = station_positions
antennaTable = solsetTEC._f_get_child('antenna')
antennaTable.append(zip(*(ants_out, ants_pos)))
# Store tecscreen values. The residual values are stored in the weights
# table
vals = tec_screen.transpose([1, 0, 2])
weights = residual.transpose([1, 0, 2])
tec_screen_st = H.makeSoltab(outSolset,
'tecscreen',
outSoltab,
axesNames=['dir', 'time', 'ant'],
axesVals=[dirs_out, times_out, ants_out],
vals=vals,
weights=weights)
# Store beta, r_0, height, and order as attributes of the tecscreen
# soltab
tec_screen_st._v_attrs['beta'] = beta
tec_screen_st._v_attrs['r_0'] = r_0
tec_screen_st._v_attrs['height'] = height
tec_screen_st._v_attrs['order'] = order
# Make output piercepoint table
tec_screen_solset = tec_screen_st._v_parent._v_name
H.H.create_carray('/' + tec_screen_solset + '/' +
tec_screen_st._v_name,
'piercepoint',
obj=pp)
# Add histories
sw = solWriter(tec_screen_st)
sw.addHistory('CREATE (by TECSCREEN operation)')
indx += 1
return 0
def run( step, parset, H ):
"""
Generic unspecified step for easy expansion.
"""
import numpy as np
from h5parm import solFetcher, solWriter
# all the following are LoSoTo function to extract information from the parset
# get involved solsets using local step values or global values or all
solsets = getParSolsets( step, parset, H )
logging.info('Solset: '+str(solsets))
# get involved soltabs using local step values or global values or all
soltabs = getParSoltabs( step, parset, H )
logging.info('Soltab: '+str(soltabs))
# get list of Antennas using local step values or global values or all
ants = getParAxis( step, parset, H, 'ant' )
logging.info('Ant: '+str(ants))
# get list of Polarizations using local step values or global values or all
pols = getParAxis( step, parset, H, 'pol' )
logging.info('Pol: '+str(pols))
# get list of SolTypes using local step values or global values or all
solTypes = getParSolTypes( step, parset, H )
logging.info('SolType: '+str(solTypes))
# get list of Directions using local step values or global values or all
dirs = getParAxis( step, parset, H, 'dir' )
logging.info('Dir: '+str(dirs))
# do something on every soltab (use the openSoltab LoSoTo function)
for soltab in openSoltabs( H, soltabs ):
logging.info("--> Working on soltab: "+soltab._v_name)
# use the solFetcher from the H5parm lib
t = solFetcher(soltab)
tw = solWriter(soltab)
axisNames = t.getAxesNames()
logging.info("Axis names are: "+str(axisNames))
solType = t.getType()
logging.info("Soltab type is: "+solType)
# this will make a selection for the getValues() and getValuesIter()
t.setSelection(ant=ants, pol=pols, dir=dirs)
logging.info("Selection is: "+str(t.selection))
# find axis values
logging.info("Antennas (no selection) are: "+str(t.getAxisValues('ant', ignoreSelection=True)))
logging.info("Antennas (with selection) are: "+str(t.getAxisValues('ant')))
# but one can also use (selection is active here!)
logging.info("Antennas (other method) are: "+str(t.ant))
logging.info("Frequencies are: "+str(t.freq))
logging.info("Directions are: "+str(t.dir))
logging.info("Polarizations are: "+str(t.pol))
# try to access a non-existent axis
t.getAxisValues('nonexistantaxis')
# now get all values given this selection
logging.info("Get data using t.val")
val = t.val
logging.debug('shape of val: '+str(t.val.shape))
logging.info("$ val is "+str(val[0,0,0,0,100]))
weight = t.weight
time = t.time
thisTime = t.time[100]
# another way to get the data is using the getValues()
logging.info("Get data using getValues()")
grid, axes = t.getValues()
# axis names
logging.info("Axes: "+str(t.getAxesNames()))
# axis shape
print axes
print [t.getAxisLen(axis) for axis in axes] # not ordered, is a dict!
# data array shape (same of axis shape)
logging.info("Shape of values: "+str(grid.shape))
#logging.info("$ val is "+str(grid[0,0,0,0,100]))
# reset selection
t.setSelection()
logging.info('Reset selection to \'\'')
logging.info("Antennas are: "+str(t.ant))
logging.info("Frequencies are: "+str(t.freq))
logging.info("Directions are: "+str(t.dir))
logging.info("Polarizations are: "+str(t.pol))
# finally the getValuesIter allaws to iterate across all possible combinations of a set of axes
logging.info('Iteration on time/freq')
for vals, coord in t.getValuesIter(returnAxes=['time','freq']):
# writing back the solutions
coord = removeKeys(coord, ['time','freq']) # removeKeys is an function of the operation_lib
tw.setSelection(**coord)
tw.setValues(vals)
logging.info('Iteration on time')
for vals, coord in t.getValuesIter(returnAxes=['time']):
# writing back the solutions
coord = removeKeys(coord, ['time','freq']) # removeKeys is an function of the operation_lib
tw.setSelection(**coord)
tw.setValues(vals)
logging.info('Iteration on dir after selection to 1 dir')
t.setSelection(dir='pointing')
for vals, coord in t.getValuesIter(returnAxes=['dir']):
# writing back the solutions
coord = removeKeys(coord, ['time','freq']) # removeKeys is an function of the operation_lib
tw.setSelection(**coord)
tw.setValues(vals)
return 0 # if everything went fine, otherwise 1
def run(step, parset, H):
import numpy as np
from h5parm import solFetcher, solWriter
soltabs = getParSoltabs(step, parset, H)
ants = getParAxis(step, parset, H, 'ant')
pols = getParAxis(step, parset, H, 'pol')
dirs = getParAxis(step, parset, H, 'dir')
axesToMed = parset.getStringVector(
'.'.join(["LoSoTo.Steps", step, "Axes"]), [])
clipLevel = parset.getFloat('.'.join(["LoSoTo.Steps", step, "SigLevel"]),
0.)
if len(axesToMed) < 1:
logging.error("Please specify axes.")
return 1
if clipLevel == 0.:
logging.error("Please specify significance level above which to clip.")
return 1
for soltab in openSoltabs(H, soltabs):
sf = solFetcher(soltab)
sw = solWriter(soltab)
logging.info("Smoothing soltab: " + soltab.name)
sf.setSelection(ant=ants, pol=pols, dir=dirs)
# some checks
if len(axesToMed) < 1:
logging.error("Please specify axes.")
return 1
if clipLevel == 0.:
logging.error(
"Please specify significance level above which to clip.")
return 1
for i, axis in enumerate(axesToMed[:]):
if axis not in sf.getAxes():
del axesToSmooth[i]
logging.warning('Axis \"' + axis + '\" not found. Ignoring.')
for vals, coord in sf.getValuesIter(returnAxes=axesToMed):
# clipping
# first find the median and standard deviation
valmedian = np.median(vals)
clipvalue = valmedian * clipLevel
np.putmask(vals, vals > clipvalue, np.nan)
# writing back the solutions
coord = removeKeys(coord, axesToSmooth)
sw.setSelection(**coord)
sw.setValues(vals)
sw.addHistory('FLAG (over %s with %s sigma cut)' %
(axesToMed, clipLevel))
return 0
def run( step, parset, H ):
"""
Fits a screen to TEC values derived by the TECFIT operation.
The TEC values are read from the specified tec-type soltab.
The results of the fit are stored in the specified tecfitwhite- and
piercepoint-type soltabs.
TEC screens can be plotted with the PLOT operation by setting PlotType =
TECScreen.
Note that the output screens are not normalized (any normalization was lost
due to the use of source-to-source phase gradients in the TECFIT operation).
Therefore, a direction-independent calibration must be done after exporting
the screens to a parmdb file, with the following settings in the BBS solve
step:
Model.Ionosphere.Enable = T
Model.Ionosphere.Type = EXPION
"""
import numpy as np
import re
from h5parm import solFetcher, solWriter
soltabs = getParSoltabs( step, parset, H )
outSoltabsTEC = parset.getStringVector('.'.join(["LoSoTo.Steps", step, "OutSoltabTEC"]), [] )
outSoltabsPP = parset.getStringVector('.'.join(["LoSoTo.Steps", step, "OutSoltabPP"]), [] )
height = np.float(parset.getString('.'.join(["LoSoTo.Steps", step, "Height"]), '200' ))
order = int(parset.getString('.'.join(["LoSoTo.Steps", step, "Order"]), '15' ))
logging.info('Using height = {0} m and order = {1}'.format(height, order))
# Load TEC values from TECFIT operation
indx = 0
for soltab in openSoltabs(H, soltabs):
if 'tec' not in soltab._v_title:
logging.warning('No TECFIT solution tables found for solution table '
'{0}'.format(soltabs[indx]))
continue
indx += 1
solset = soltabs[indx].split('/')[0]
logging.info('Using input solution table: {0}'.format(soltabs[indx]))
logging.info('Using output solution tables: {0} and {1}'.format(
outSoltabsTEC[indx], outSoltabsPP[indx]))
station_dict = H.getAnt(solset)
station_names = station_dict.keys()
station_positions = station_dict.values()
source_dict = H.getSou(solset)
source_names = source_dict.keys()
source_positions = source_dict.values()
t = solFetcher(soltab)
r, axis_vals = t.getValues()
times = axis_vals['time']
N_sources = len(source_names)
N_times = len(times)
N_stations = len(station_names)
N_piercepoints = N_sources * N_stations
rr = reshape(r.transpose([0,2,1]), [N_piercepoints, N_times])
# Find pierce points and airmass values for given screen height
pp, airmass = calculate_piercepoints(np.array(station_positions),
np.array(source_positions), np.array(times), height)
# Fit a TEC screen
r_0 = 10e3
beta = 5.0/3.0
tec_fit, tec_fit_white = fit_screen_to_tec(station_names, source_names,
pp, airmass, rr, times, height, order, r_0, beta)
# Write the results to the output solset
dirs_out = []
dirs_pos = []
for s in range(N_sources):
dirs_out.append(source_names[s])
times_out = times
ants_out = []
ants_pos = []
for s in range(N_stations):
ants_out.append(station_names[s])
# Make output tecfitwhite table
outSolsetTEC = outSoltabsTEC[indx].split('/')[0]
outSoltabTEC = outSoltabsTEC[indx].split('/')[1]
if not outSolsetTEC in H.getSolsets().keys():
solsetTEC = H.makeSolset(outSolsetTEC)
dirs_out = []
dirs_pos = []
for s in range(N_sources):
dirs_out.append(source_names[s])
dirs_pos.append(source_positions[s])
sourceTable = solsetTEC._f_get_child('source')
sourceTable.append(zip(*(dirs_out, dirs_pos)))
ants_out = []
ants_pos = []
for s in range(N_stations):
ants_out.append(station_names[s])
ants_pos.append(station_positions[s])
antennaTable = solsetTEC._f_get_child('antenna')
antennaTable.append(zip(*(ants_out, ants_pos)))
# Store tecfitwhite values. The tec_fit values are stored in the weights
# table to simplify things
vals = tec_fit_white.transpose([1, 0, 2])
weights = tec_fit.transpose([1, 0, 2])
tec_fit_st = H.makeSoltab(outSolsetTEC, 'tecfitwhite', outSoltabTEC,
axesNames=['dir', 'time', 'ant'], axesVals=[dirs_out, times_out,
ants_out], vals=vals, weights=weights)
# Store beta, r_0, height, and order as attributes of the tecfitwhite
# soltab
tec_fit_st._v_attrs['beta'] = beta
tec_fit_st._v_attrs['r_0'] = r_0
tec_fit_st._v_attrs['height'] = height
tec_fit_st._v_attrs['order'] = order
# Make output piercepoint table
outSolsetPP = outSoltabsPP[indx].split('/')[0]
outSoltabPP = outSoltabsPP[indx].split('/')[1]
if not outSolsetPP in H.getSolsets().keys():
solsetPP = H.makeSolset(outSolsetPP)
sourceTable = solsetPP._f_get_child('source')
sourceTable.append(zip(*(dirs_out, dirs_pos)))
antennaTable = solsetPP._f_get_child('antenna')
antennaTable.append(zip(*(ants_out, ants_pos)))
# Store piercepoint positions
pp_indx = range(N_piercepoints)
pp_pos_indx = range(3) # 0 -> x, 1 -> y, 2 -> z
pp_st = H.makeSoltab(outSolsetPP, 'piercepoint', outSoltabPP,
axesNames=['time', 'piercepoint', 'coord'], axesVals=[times_out, pp_indx,
pp_pos_indx], vals=pp, weights=np.ones_like(pp))
# Add histories
sw = solWriter(tec_fit_st)
sw.addHistory('CREATE (by TECSCREEN operation)')
sw = solWriter(pp_st)
sw.addHistory('CREATE (by TECSCREEN operation)')
indx += 1
return 0
def run(step, parset, H):
import scipy.ndimage.filters
from h5parm import solFetcher, solWriter
soltabs = getParSoltabs(step, parset, H)
ants = getParAxis(step, parset, H, 'ant')
pols = getParAxis(step, parset, H, 'pol')
dirs = getParAxis(step, parset, H, 'dir')
axesToSmooth = parset.getStringVector(
'.'.join(["LoSoTo.Steps", step, "Axes"]), [])
FWHM = parset.getIntVector('.'.join(["LoSoTo.Steps", step, "FWHM"]), [])
if len(axesToSmooth) != len(FWHM):
logging.error("Axes and FWHM lenghts must be equal.")
return 1
for soltab in openSoltabs(H, soltabs):
sf = solFetcher(soltab)
sw = solWriter(soltab)
logging.info("Smoothing soltab: " + soltab._v_name)
sf.setSelection(ant=ants, pol=pols, dir=dirs)
for i, axis in enumerate(axesToSmooth[:]):
if axis not in sf.getAxesNames():
del axesToSmooth[i]
del FWHM[i]
logging.warning('Axis \"' + axis + '\" not found. Ignoring.')
for vals, coord in sf.getValuesIter(returnAxes=axesToSmooth):
valsnew = scipy.ndimage.filters.median_filter(vals, FWHM)
# TODO: implement flag control
# find the local mean (without any nans)
# valmean = np.ma.mean(np.ma.masked_array(vals, np.isnan(vals)), axis=0)
# print 'mean value: ' + str(valmean)
# replace any nans with median
# valbool = np.isnan(vals)
# nanindex = np.where(valbool)
# if valbool.any():
# logging.warning('Found NaNs in solutions. Replacing with local mean.')
# check if valmean is iterable
# if valmean.shape is tuple():
# np.putmask(vals, valbool, valmean)
# else:
# for x,mval in enumerate(valmean):
# np.putmask(vals[:,x], valbool[:,x], mval)
# writing back the solutions (selection on all the coord axis)
# this can be properly broacasted
coord = removeKeys(coord, axesToSmooth)
sw.setSelection(**coord)
sw.setValues(valsnew)
sw.addHistory('SMOOTH (over %s with box size = %s)' %
(axesToSmooth, FWHM))
return 0
def run( step, parset, H ):
"""
Fit phase solutions to obtain TEC values per direction.
Phase solutions are assumed to be stored in solsets of the H5parm file, one
solset per band per field. Only phase- or scalarphase-type solution tables
are used. If direction-independent solutions are found (in addition to the
direction-dependent ones), they are added, after averaging, to the
corresponding direction-dependent ones. Phase solutions are automatically
grouped by field and by band.
The derived TEC values are stored in the specified output soltab of type
'tec', with one TEC value per station per direction per solution interval.
The TEC values are derived using the ``lofar.expion.baselinefitting.fit()``
function to fit a TEC value to the phases. The term that is minimized
includes all baselines, so there is no preferred reference station, and the
residual is computed as the complex number 1.0 - exp(1i phasedifference),
which is zero when the phase difference is a multiple of 2pi.
The TEC solution table may be used to derive TEC screens using the
TECSCREEN operation.
"""
from h5parm import solFetcher, solWriter
import numpy as np
from pylab import find
import re
solsets = getParSolsets( step, parset, H )
ants = getParAxis( step, parset, H, 'ant' )
pols = getParAxis( step, parset, H, 'pol' )
dirs = getParAxis( step, parset, H, 'dir' )
nband_min = int(parset.getString('.'.join(["LoSoTo.Steps", step, "MinBands"]), '8' ))
dist_cut_m = np.float(parset.getString('.'.join(["LoSoTo.Steps", step, "DistCut"]), '2e3' ))
nstations_max = int(parset.getString('.'.join(["LoSoTo.Steps", step, "MaxStations"]), '100' ))
outSolset = parset.getString('.'.join(["LoSoTo.Steps", step, "OutSoltab"]), '' ).split('/')[0]
outSoltab = parset.getString('.'.join(["LoSoTo.Steps", step, "OutSoltab"]), '' ).split('/')[1]
# Collect solutions, etc. into arrays for fitting.
(phases0, phases1, flags, mask, station_names, station_positions,
source_names, source_positions, freqs, times, pointing,
soln_type) = collect_solutions(H, dirs=dirs, solsets=solsets)
if phases0 is None:
return 1
# Build list of stations to include
included_stations = []
if ants is not None:
for ant in ants:
included_stations += [s for s in station_names if re.search(ant, s)]
else:
included_stations = station_names
excluded_stations = [s for s in station_names if s not in included_stations]
# Select stations to use for first pass
if len(excluded_stations) > 0:
logging.info("Excluding stations: {0}".format(np.sort(excluded_stations)))
mean_position = np.array([np.median(station_positions[:, 0]),
np.median(station_positions[:, 1]), np.median(station_positions[:, 2])])
dist = np.sqrt(np.sum((station_positions - mean_position)**2, axis=1))
dist_sort_ind = np.argsort(dist)
station_selection1 = find(dist < dist_cut_m)
station_selection = np.array([i for i in dist_sort_ind
if i in station_selection1 and station_names[i] not in excluded_stations])
if len(station_selection) > nstations_max:
station_selection = station_selection[:nstations_max]
logging.info("Using normal fitting (no iterative search) for {0} stations "
"within {1} km of the core:\n{2}".format(len(station_selection),
dist_cut_m/1000.0, station_names[station_selection]))
# Fit a TEC value to the phase solutions per source pair. No iterative search for the
# global minimum is done
if soln_type == 'scalarphase':
r, source_selection = fit_tec_per_source_pair(
phases0[:, station_selection, :, :],
flags[:, station_selection, :, :],
mask, freqs, propagate=True, nband_min=nband_min)
if r is None:
return 1
else:
r0, source_selection = fit_tec_per_source_pair(
phases0[:, station_selection, :, :],
flags[:, station_selection, :, :],
mask, freqs, propagate=True, nband_min=nband_min)
r1, source_selection = fit_tec_per_source_pair(
phases1[:, station_selection, :, :],
flags[:, station_selection, :, :],
mask, freqs, propagate=True, nband_min=nband_min)
if r0 is None or r1 is None:
return 1
# take the mean of the two polarizations
r = (r0 + r1) / 2
# Add stations by searching iteratively for global minimum in solution space
station_selection, r = add_stations(station_selection, phases0,
phases1, flags, mask, station_names, station_positions, source_names,
source_selection, times, freqs, r, nband_min=nband_min,
soln_type=soln_type, nstations_max=nstations_max, excluded_stations=
excluded_stations, search_full_tec_range=False)
# Save TEC values to the output solset
solset = H.makeSolset(outSolset)
dirs_out = []
dirs_pos = []
for s in source_selection:
dirs_out.append(source_names[s])
dirs_pos.append(source_positions[s])
sourceTable = solset._f_get_child('source')
sourceTable.append(zip(*(dirs_out, dirs_pos)))
times_out = times
ants_out = []
ants_pos = []
for s in station_selection:
ants_out.append(station_names[s])
ants_pos.append(station_positions[s])
antennaTable = solset._f_get_child('antenna')
antennaTable.append(zip(*(ants_out, ants_pos)))
tf_st = H.makeSoltab(solset._v_name, 'tec', outSoltab,
axesNames=['dir', 'time', 'ant'], axesVals=[dirs_out, times, ants_out],
vals=r[source_selection, :, :],
weights=np.ones_like(r[source_selection, :, :]))
# Add history
sw = solWriter(tf_st)
sw.addHistory('CREATE (by TECFIT operation)')
return 0
def run( step, parset, H ):
"""
Generic unspecified step for easy expansion.
"""
import numpy as np
from h5parm import solFetcher, solWriter
# all the following are LoSoTo function to extract information from the parset
# get involved solsets using local step values or global values or all
solsets = getParSolsets( step, parset, H )
logging.info('Solset: '+str(solsets))
# get involved soltabs using local step values or global values or all
soltabs = getParSoltabs( step, parset, H )
logging.info('Soltab: '+str(soltabs))
# get list of Antennas using local step values or global values or all
ants = getParAxis( step, parset, H, 'ant' )
logging.info('Ant: '+str(ants))
# get list of Polarizations using local step values or global values or all
pols = getParAxis( step, parset, H, 'pol' )
logging.info('Pol: '+str(pols))
# get list of SolTypes using local step values or global values or all
solTypes = getParSolTypes( step, parset, H )
logging.info('SolType: '+str(solTypes))
# get list of Directions using local step values or global values or all
dirs = getParAxis( step, parset, H, 'dir' )
logging.info('Dir: '+str(dirs))
# do something on every soltab (use the openSoltab LoSoTo function)
for soltab in openSoltabs( H, soltabs ):
logging.info("--> Working on soltab: "+soltab._v_name)
# use the solFetcher from the H5parm lib
t = solFetcher(soltab)
tw = solWriter(soltab)
axisNames = t.getAxesNames()
logging.info("Axis names are: "+str(axisNames))
solType = t.getType()
logging.info("Soltab type is: "+solType)
# this will make a selection for the getValues() and getValuesIter()
t.setSelection(ant=ants, pol=pols, dir=dirs)
logging.info("Selection is: "+str(t.selection))
# find all axis values
logging.info("Antennas are: "+str(t.getAxisValues('ant')))
# but one can also use
logging.info("Antennas (other method) are: "+str(t.ant))
logging.info("Frequencies are: "+str(t.freq))
logging.info("Directions are: "+str(t.dir))
logging.info("Polarizations are: "+str(t.pol))
# try to access a non-existent axis
t.getAxisValues('nonexistantaxis')
# now get all values given this selection
logging.info("Get data using t.val")
val = t.val
logging.info("$ val is "+str(val[0,0,0,0,100]))
flag = t.weight
time = t.time
thisTime = t.time[100]
# another way to get the data is using the getValues()
logging.info("Get data using getValues()")
grid, axes = t.getValues()
# axis names
logging.info("Axes: "+str(t.getAxesNames()))
# axis shape
print axes
print [t.getAxisLen(axis) for axis in axes] # not ordered, is a dict!
def run( step, parset, H ):
"""
Separate phase solutions into Clock and TEC.
Phase solutions are assumed to be stored in solsets of the H5parm file, one
solset per field.
The Clock and TEC values are stored in the specified output soltab with type 'clock' and 'tec'.
"""
import numpy as np
from h5parm import solFetcher, solWriter
ants = getParAxis( step, parset, H, 'ant' )
logging.info('Ant: '+str(ants))
pols = getParAxis( step, parset, H, 'pol' )
logging.info('Pol: '+str(pols))
dirs = getParAxis( step, parset, H, 'dir' )
logging.info('Dir: '+str(dirs))
# get involved solsets using local step values or global values or all
solsets = getParSolsets( step, parset, H )
logging.info('Solset: '+str(solsets))
soltabs = getParSoltabs( step, parset, H )
logging.info('Soltab: '+str(soltabs))
solTypes = ['phase']
# do something on every soltab (use the openSoltab LoSoTo function)
#for soltab in openSoltabs( H, soltabs ):
for soltabname in soltabs:
solsetname=soltabname.split('/')[0]
soltab=H.getSoltab(solset=solsetname, soltab=soltabname.split('/')[1])
logging.info("--> Working on soltab: "+soltab._v_name)
# use the solFetcher from the H5parm lib
t = solFetcher(soltab)
tw = solWriter(soltab)
axisNames = t.getAxesNames()
logging.info("Axis names are: "+str(axisNames))
solType = t.getType()
if solType != 'phase':
logging.info("Soltab type of "+soltab._v_name+" is: "+solType," should be phase")
continue
# this will make a selection for the getValues() and getValuesIter()
t.setSelection(ant=ants, pol=pols, dir=dirs)
logging.info("Selection is: "+str(t.selection))
names=t.getAxesNames()
logging.info("axis names"+str(t.getAxesNames()))
# Collect station properties
station_dict = H.getAnt(solsetname)
stations=t.getAxisValues('ant')
station_positions = np.zeros((len(stations), 3), dtype=np.float)
for i, station_name in enumerate(stations):
station_positions[i, 0] = station_dict[station_name][0]
station_positions[i, 1] = station_dict[station_name][1]
station_positions[i, 2] = station_dict[station_name][2]
returnAxes=['ant','freq','pol','time']
for vals, coord in t.getValuesIter(returnAxes=returnAxes):
freqs=coord['freq']
ph=vals[:]
stations=coord['ant']
axes=[i for i in names if i in returnAxes]
clock,tec,offset,newstations=doFit(ph,freqs,stations,station_positions,axes)
times=coord['time']
tf_st = H.makeSoltab(solsetname, 'tec', 'tec',
axesNames=['time', 'ant','pol'], axesVals=[times, newstations,np.arange(2)],
vals=tec,
weights=np.ones_like(tec))
sw = solWriter(tf_st)
sw.addHistory('CREATE (by CLOCKTECFIT operation)')
tf_st = H.makeSoltab(solsetname, 'clock', 'clock',
axesNames=['time', 'ant','pol'], axesVals=[times, newstations,np.arange(2)],
vals=clock,
weights=np.ones_like(clock))
sw = solWriter(tf_st)
sw.addHistory('CREATE (by CLOCKTECFIT operation)')
tf_st = H.makeSoltab(solsetname, 'offset', 'phase_offset',
axesNames=['ant','pol'], axesVals=[newstations,np.arange(2)],
vals=offset,
weights=np.ones_like(offset))
sw = solWriter(tf_st)
sw.addHistory('CREATE (by CLOCKTECFIT operation)')
# Add history
return 0 # if everything went fine, otherwise 1
def run( step, parset, H ):
"""
Fit source-to-source phase gradients to obtain TEC values per direction.
Phase solutions are assumed to be stored in solsets of the H5parm file, one
solset per band per field.
The TEC values are stored in the specified output soltab with type 'tec'.
"""
import numpy as np
from h5parm import solFetcher, solWriter
ants = getParAxis( step, parset, H, 'ant' )
pols = getParAxis( step, parset, H, 'pol' )
dirs = getParAxis( step, parset, H, 'dir' )
nband_min = int(parset.getString('.'.join(["LoSoTo.Steps", step, "MinBands"]), '8' ))
nstations_max = int(parset.getString('.'.join(["LoSoTo.Steps", step, "MaxStations"]), '100' ))
outSolset = parset.getString('.'.join(["LoSoTo.Steps", step, "OutSoltab"]), '' ).split('/')[0]
outSoltab = parset.getString('.'.join(["LoSoTo.Steps", step, "OutSoltab"]), '' ).split('/')[1]
# Collect solutions, etc. into arrays for fitting.
# At the moment, all solsets of H are searched for phase solutions.
phases0, phases1, flags, mask, station_names, station_positions, source_names, source_positions, freqs, times, pointing, soln_type = collect_solutions(H, dirs=dirs)
# Build list of stations to include
included_stations = []
if ants is not None:
for ant in ants:
included_stations += [s for s in station_names if re.search(ant, s)]
else:
included_stations = station_names
excluded_stations = [s for s in station_names if s not in included_stations]
# Select stations to use for first pass
dist_cut_m = 2e3
logging.info("Excluding stations: {0}".format(sort(excluded_stations)))
mean_position = np.array([median(station_positions[:, 0]), median(station_positions[:, 1]), median(station_positions[:, 2])])
station_selection1 = find(sqrt(sum((station_positions - mean_position)**2, axis=1)) < dist_cut_m)
station_selection = array([i for i in range(len(station_names)) if i in station_selection1 and station_names[i] not in excluded_stations])
# Fit a TEC value to the phase solutions per source pair. No search for the
# global minimum is done
search_tec = True
nsteps = 21
if soln_type == 'scalarphase':
r, source_selection, sols0, source_pairs = fit_tec_per_source_pair(phases0[:,station_selection,:,:], flags[:,station_selection,:,:], mask, freqs, propagate = True, nband_min=nband_min)
if r is None:
return 1
else:
r0, source_selection, sols0, source_pairs = fit_tec_per_source_pair(phases0[:,station_selection,:,:], flags[:,station_selection,:,:], mask, freqs, propagate = True, nband_min=nband_min)
r1, source_selection, sols1, source_pairs = fit_tec_per_source_pair(phases1[:,station_selection,:,:], flags[:,station_selection,:,:], mask, freqs, propagate = True, nband_min=nband_min)
if r0 is None or r1 is None:
return 1
# take the mean of the two polarizations
r = (r0+r1)/2
# Add stations by searching for global minimum in solution space
station_selection, sols, r = add_stations(station_selection, phases0, phases1, flags, mask, station_names, station_positions, source_names, source_selection, times, freqs, r, nband_min=nband_min, soln_type=soln_type, nstations_max=nstations_max)
# Save TEC values to the output solset
solset = H.makeSolset(outSolset)
dirs_out = []
dirs_pos = []
for s in source_selection:
dirs_out.append(source_names[s])
dirs_pos.append(source_positions[s])
sourceTable = solset._f_get_child('source')
sourceTable.append(zip(*(dirs_out, dirs_pos)))
times_out = times
ants_out = []
ants_pos = []
for s in station_selection:
ants_out.append(station_names[s])
ants_pos.append(station_positions[s])
antennaTable = solset._f_get_child('antenna')
antennaTable.append(zip(*(ants_out, ants_pos)))
tf_st = H.makeSoltab(outSolset, 'tec', outSoltab, axesNames=['dir', 'time', 'ant'],
axesVals=[dirs_out, times, ants_out], vals=r, weights=np.ones_like(r))
# Add history
sw = solWriter(tf_st)
sw.addHistory('CREATE (by TECFIT operation)')
return 0
def run( step, parset, H ):
"""
Fits a screen to TEC values derived by the TECFIT operation.
The TEC values are read from the specified tec soltab.
The results of the fit are stored in the specified tecscreen solution table.
These values are the screen TEC values per station per pierce point per
solution interval. The pierce point locations are stored in an auxiliary
array in the output solution table.
TEC screens can be plotted with the PLOT operation by setting PlotType =
TECScreen.
The H5parm_exporter.py tool can be used to export the screen to a parmdb
that BBS and the AWimager can use. Note, however, that the output screens
are not normalized properly (any normalization was lost due to the use of
source-to-source phase gradients in the TECFIT operation). Therefore, a
direction-independent calibration must be done after exporting the screens
to a parmdb file, with the following settings in the BBS solve step:
Model.Ionosphere.Enable = T
Model.Ionosphere.Type = EXPION
"""
import numpy as np
import re
from h5parm import solFetcher, solWriter
# Switch to the Agg backend to prevent problems with pylab imports when
# DISPLAY env. variable is not set
import os
if 'DISPLAY' not in os.environ:
import matplotlib
matplotlib.use("Agg")
soltabs = getParSoltabs( step, parset, H )
outSoltabs = parset.getStringVector('.'.join(["LoSoTo.Steps", step, "OutSoltab"]), [] )
height = np.array(parset.getDoubleVector('.'.join(["LoSoTo.Steps", step, "Height"]), [200e3] ))
order = int(parset.getString('.'.join(["LoSoTo.Steps", step, "Order"]), '15' ))
# Load TEC values from TECFIT operation
indx = 0
for soltab in openSoltabs(H, soltabs):
if 'tec' not in soltab._v_title:
logging.warning('No TECFIT solution tables found for solution table '
'{0}'.format(soltabs[indx]))
continue
indx += 1
solset = soltabs[indx].split('/')[0]
logging.info('Using input solution table: {0}'.format(soltabs[indx]))
logging.info('Using output solution table: {0}'.format(outSoltabs[indx]))
# Collect station and source names and positions and times, making sure
# that they are ordered correctly.
t = solFetcher(soltab)
r, axis_vals = t.getValues()
source_names = axis_vals['dir']
source_dict = H.getSou(solset)
source_positions = []
for source in source_names:
source_positions.append(source_dict[source])
station_names = axis_vals['ant']
station_dict = H.getAnt(solset)
station_positions = []
for station in station_names:
station_positions.append(station_dict[station])
times = axis_vals['time']
# Get sizes
N_sources = len(source_names)
N_times = len(times)
N_stations = len(station_names)
N_piercepoints = N_sources * N_stations
rr = np.reshape(r.transpose([0, 2, 1]), [N_piercepoints, N_times])
heights = list(set(np.linspace(height[0], height[-1], 5)))
heights.sort()
if len(heights) > 1:
logging.info('Trying range of heights: {0} m'.format(heights))
for i, height in enumerate(heights):
# Find pierce points and airmass values for given screen height
logging.info('Using height = {0} m and order = {1}'.format(height, order))
if height < 100e3:
logging.warning("Height is less than 100e3 m.")
pp, airmass = calculate_piercepoints(np.array(station_positions),
np.array(source_positions), np.array(times), height)
# Fit a TEC screen
r_0 = 10e3
beta = 5.0 / 3.0
tec_screen, residual = fit_screen_to_tec(station_names, source_names,
pp, airmass, rr, times, height, order, r_0, beta)
total_resid = np.sum(np.abs(residual))
if i > 0:
if total_resid < best_resid:
tec_screen_best = tec_screen
pp_best = pp
height_best = height
best_resid = total_resid
else:
tec_screen_best = tec_screen
pp_best = pp
height_best = height
best_resid = total_resid
if len(heights) > 1:
logging.info('Total residual for fit: {0}\n'.format(total_resid))
# Use screen with lowest total residual
if len(heights) > 1:
tec_screen = tec_screen_best
pp = pp_best
height = height_best
logging.info('Using height (with lowest total residual) of {0} m'.format(height))
# Write the results to the output solset
dirs_out = source_names
times_out = times
ants_out = station_names
# Make output tecscreen table
outSolset = outSoltabs[indx].split('/')[0]
outSoltab = outSoltabs[indx].split('/')[1]
if not outSolset in H.getSolsets().keys():
solsetTEC = H.makeSolset(outSolset)
dirs_pos = source_positions
sourceTable = solsetTEC._f_get_child('source')
sourceTable.append(zip(*(dirs_out, dirs_pos)))
ants_pos = station_positions
antennaTable = solsetTEC._f_get_child('antenna')
antennaTable.append(zip(*(ants_out, ants_pos)))
# Store tecscreen values. The residual values are stored in the weights
# table. Flagged values of the screen have weights set to 0.0.
vals = tec_screen.transpose([1, 0, 2])
weights = residual.transpose([1, 0, 2])
tec_screen_st = H.makeSoltab(outSolset, 'tecscreen', outSoltab,
axesNames=['dir', 'time', 'ant'], axesVals=[dirs_out, times_out,
ants_out], vals=vals, weights=weights)
# Store beta, r_0, height, and order as attributes of the tecscreen
# soltab
tec_screen_st._v_attrs['beta'] = beta
tec_screen_st._v_attrs['r_0'] = r_0
tec_screen_st._v_attrs['height'] = height
tec_screen_st._v_attrs['order'] = order
# Make output piercepoint table
tec_screen_solset = tec_screen_st._v_parent._v_name
H.H.create_carray('/'+tec_screen_solset+'/'+tec_screen_st._v_name,
'piercepoint', obj=pp)
# Add histories
sw = solWriter(tec_screen_st)
sw.addHistory('CREATE (by TECSCREEN operation)')
indx += 1
return 0
def run(step, parset, H):
"""
Interpolate the solutions from one table into a destination table
"""
import itertools
import scipy.interpolate
import numpy as np
from h5parm import solFetcher, solWriter
solsets = getParSolsets(step, parset, H)
soltabs = getParSoltabs(step, parset, H)
solTypes = getParSolTypes(step, parset, H)
ants = getParAxis(step, parset, H, 'ant')
pols = getParAxis(step, parset, H, 'pol')
dirs = getParAxis(step, parset, H, 'dir')
calSoltab = parset.getString('.'.join(["LoSoTo.Steps", step, "CalSoltab"]),
'')
calDir = parset.getString('.'.join(["LoSoTo.Steps", step, "CalDir"]), '')
interpAxes = parset.getStringVector(
'.'.join(["LoSoTo.Steps", step, "InterpAxes"]), ['time', 'freq'])
interpMethod = parset.getString(
'.'.join(["LoSoTo.Steps", step, "InterpMethod"]), 'linear')
medAxis = parset.getString('.'.join(["LoSoTo.Steps", step, "MedAxis"]), '')
rescale = parset.getBool('.'.join(["LoSoTo.Steps", step, "Rescale"]),
False)
if interpMethod not in ["nearest", "linear", "cubic"]:
logging.error('Interpolation method must be nearest, linear or cubic.')
return 1
if rescale and medAxis == '':
logging.error('A medAxis is needed for rescaling.')
return 1
# open calibration table
css, cst = calSoltab.split('/')
cr = solFetcher(H.getSoltab(css, cst))
cAxesNames = cr.getAxesNames()
for soltab in openSoltabs(H, soltabs):
logging.info("--> Working on soltab: " + soltab._v_name)
tr = solFetcher(soltab)
tw = solWriter(soltab)
axesNames = tr.getAxesNames()
for i, interpAxis in enumerate(interpAxes[:]):
if interpAxis not in axesNames or interpAxis not in cAxesNames:
logging.error('Axis ' + interpAxis + ' not found. Ignoring.')
del interpAxes[i]
if medAxis not in axesNames or medAxis not in cAxesNames:
logging.error('Axis ' + medAxis + ' not found. Cannot proceed.')
return 1
tr.setSelection(ant=ants, pol=pols, dir=dirs)
for vals, coord in tr.getValuesIter(returnAxes=interpAxes):
# construct grid
coordSel = removeKeys(coord, interpAxes)
logging.debug("Working on coords:" + str(coordSel))
# change dir if sepcified
if calDir != '':
coordSel['dir'] = calDir
cr.setSelection(**coordSel)
calValues, calCoord = cr.getValues()
# fill medAxis with the median value
axis = cAxesNames.index(medAxis)
calValues = np.repeat(
np.expand_dims(np.median(calValues, axis), axis),
calValues.shape[axis], axis)
# create a list of values whose coords are calPoints
calValues = np.ndarray.flatten(calValues)
# create calibrator/target coordinates arrays
calPoints = []
targetPoints = []
for interpAxis in interpAxes:
calPoints.append(calCoord[interpAxis])
targetPoints.append(coord[interpAxis])
calPoints = np.array([x for x in itertools.product(*calPoints)])
targetPoints = np.array(
[x for x in itertools.product(*targetPoints)])
# interpolation
valsNew = scipy.interpolate.griddata(calPoints, calValues,
targetPoints, interpMethod)
# fill values outside boudaries with "nearest" solutions
if interpMethod != 'nearest':
valsNewNearest = scipy.interpolate.griddata(
calPoints, calValues, targetPoints, 'nearest')
valsNew[np.where(valsNew == np.nan)] = valsNewNearest[np.where(
valsNew == np.nan)]
valsNew = valsNew.reshape(vals.shape)
if rescale:
# rescale solutions
axis = interpAxes.index(medAxis)
valsMed = np.repeat(
np.expand_dims(np.median(vals, axis), axis),
vals.shape[axis], axis)
valsNewMed = np.repeat(
np.expand_dims(np.median(valsNew, axis), axis),
valsNew.shape[axis], axis)
valsNew = vals * valsNewMed / valsMed
#print "Rescaling by: ", valsNewMed[:,0]/valsMed[:,0]
# writing back the solutions
coord = removeKeys(coord, interpAxes)
tw.setSelection(**coord)
tw.setValues(valsNew)
tw.addHistory('INTERP (from table %s)' % (calSoltab))
return 0