def run( step, parset, H ):
"""
Copy values from a table to another (of the same kind)
If tables have different sampling, then resample the values
"""
import numpy as np
from h5parm import solFetcher, solWriter
inTable = parset.getString('.'.join(["LoSoTo.Steps", step, "InTable"]), '' )
outTable = parset.getString('.'.join(["LoSoTo.Steps", step, "OutTable"]), '' )
tr = solFetcher(inTable)
inVals, inCoord = tr.getValues()
tw = solWriter(outTable)
outVals, outCoord = tw.getValues()
shapeDiff = tuple(np.array(outVals.shape)/np.array(inVals.shape))
inValsNew = np.kron(inVals, np.ones(shapeDiff))
if inValsNew.shape != outVals.shape:
logging.error("Incompatible shapes for copying solutions. The outTable axes can be a multiple of the inTable axis dimension.")
# writing on the output table
tw.setValues(inValsNew)
tw.addHistory('COPY (from table %s)' % (inTable))
return 0
def run(step, parset, H):
"""
Copy values from a table to another (of the same kind)
If tables have different sampling, then resample the values
"""
import numpy as np
from h5parm import solFetcher, solWriter
inTable = parset.getString('.'.join(["LoSoTo.Steps", step, "InTable"]), '')
outTable = parset.getString('.'.join(["LoSoTo.Steps", step, "OutTable"]),
'')
tr = solFetcher(inTable)
inVals, inCoord = tr.getValues()
tw = solWriter(outTable)
outVals, outCoord = tw.getValues()
shapeDiff = tuple(np.array(outVals.shape) / np.array(inVals.shape))
inValsNew = np.kron(inVals, np.ones(shapeDiff))
if inValsNew.shape != outVals.shape:
logging.error(
"Incompatible shapes for copying solutions. The outTable axes can be a multiple of the inTable axis dimension."
)
# writing on the output table
tw.setValues(inValsNew)
tw.addHistory('COPY (from table %s)' % (inTable))
return 0
def run( step, parset, H ):
import scipy.ndimage.filters
from h5parm import solFetcher, solWriter
soltabs = getParSoltabs( step, parset, H )
ants = getParAxis( step, parset, H, 'ant' )
pols = getParAxis( step, parset, H, 'pol' )
dirs = getParAxis( step, parset, H, 'dir' )
axesToSmooth = parset.getStringVector('.'.join(["LoSoTo.Steps", step, "Axes"]), [] )
FWHM = parset.getIntVector('.'.join(["LoSoTo.Steps", step, "FWHM"]), [] )
if len(axesToSmooth) != len(FWHM):
logging.error("Axes and FWHM lenghts must be equal.")
return 1
for soltab in openSoltabs( H, soltabs ):
sf = solFetcher(soltab)
sw = solWriter(soltab)
logging.info("Smoothing soltab: "+soltab._v_name)
sf.setSelection(ant=ants, pol=pols, dir=dirs)
for i, axis in enumerate(axesToSmooth[:]):
if axis not in sf.getAxesNames():
del axesToSmooth[i]
del FWHM[i]
logging.warning('Axis \"'+axis+'\" not found. Ignoring.')
for vals, coord in sf.getValuesIter(returnAxes=axesToSmooth):
valsnew = scipy.ndimage.filters.median_filter(vals, FWHM)
# TODO: implement flag control
# find the local mean (without any nans)
# valmean = np.ma.mean(np.ma.masked_array(vals, np.isnan(vals)), axis=0)
# print 'mean value: ' + str(valmean)
# replace any nans with median
# valbool = np.isnan(vals)
# nanindex = np.where(valbool)
# if valbool.any():
# logging.warning('Found NaNs in solutions. Replacing with local mean.')
# check if valmean is iterable
# if valmean.shape is tuple():
# np.putmask(vals, valbool, valmean)
# else:
# for x,mval in enumerate(valmean):
# np.putmask(vals[:,x], valbool[:,x], mval)
# writing back the solutions (selection on all the coord axis)
# this can be properly broacasted
coord = removeKeys(coord, axesToSmooth)
sw.setSelection(**coord)
sw.setValues(valsnew)
sw.addHistory('SMOOTH (over %s with box size = %s)' % (axesToSmooth, FWHM))
return 0
def run( step, parset, H ):
import scipy.ndimage.filters
from h5parm import solFetcher, solWriter
soltabs = getParSoltabs( step, parset, H )
ants = getParAxis( step, parset, H, 'ant' )
pols = getParAxis( step, parset, H, 'pol' )
dirs = getParAxis( step, parset, H, 'dir' )
axesToSmooth = parset.getStringVector('.'.join(["LoSoTo.Steps", step, "Axes"]), [] )
FWHM = parset.getIntVector('.'.join(["LoSoTo.Steps", step, "FWHM"]), [] )
if len(axesToSmooth) != len(FWHM):
logging.error("Axes and FWHM lenghts must be equal.")
return 1
for soltab in openSoltabs( H, soltabs ):
sf = solFetcher(soltab)
sw = solWriter(soltab)
logging.info("Smoothing soltab: "+soltab._v_name)
sf.setSelection(ant=ants, pol=pols, dir=dirs)
for i, axis in enumerate(axesToSmooth[:]):
if axis not in sf.getAxesNames():
del axesToSmooth[i]
del FWHM[i]
logging.warning('Axis \"'+axis+'\" not found. Ignoring.')
for vals, coord in sf.getValuesIter(returnAxes=axesToSmooth):
valsnew = scipy.ndimage.filters.median_filter(vals, FWHM)
# Assume 1D filtering
# window=FWHM[0]/2
# valsnew=np.zeros(len(vals))
# for i in xrange(0,len(vals)):
# min=i-window;
# if (min<0):
# min=0;
# max=i+window;
# if (max>=len(vals)):
# max=len(vals)-1
# valsnew[i]=np.median(vals[min:max])
coord = removeKeys(coord, axesToSmooth)
sw.setSelection(**coord)
sw.setValues(valsnew)
sw.addHistory('SMOOTH (over %s with box size = %s)' % (axesToSmooth, FWHM))
return 0
def run(step, parset, H):
"""
Normalize the solutions to a given value
"""
import numpy as np
from h5parm import solFetcher, solWriter
solsets = getParSolsets(step, parset, H)
soltabs = getParSoltabs(step, parset, H)
solTypes = getParSolTypes(step, parset, H)
ants = getParAxis(step, parset, H, 'ant')
pols = getParAxis(step, parset, H, 'pol')
dirs = getParAxis(step, parset, H, 'dir')
normVal = parset.getFloat('.'.join(["LoSoTo.Steps", step, "NormVal"]), 1.)
normAxis = parset.getString('.'.join(["LoSoTo.Steps", step, "NormAxis"]),
'time')
for soltab in openSoltabs(H, soltabs):
logging.info("--> Working on soltab: " + soltab._v_name)
tr = solFetcher(soltab)
tw = solWriter(soltab)
axesNames = tr.getAxesNames()
if normAxis not in axesNames:
logging.error('Normalization axis ' + normAxis + ' not found.')
return 1
tr.setSelection(ant=ants, pol=pols, dir=dirs)
for vals, coord in tr.getValuesIter(returnAxes=normAxis):
# construct grid
coordSel = removeKeys(coord, normAxis)
logging.debug("Working on coords:" + str(coordSel))
# rescale solutions
valsMean = np.mean(vals)
valsNew = normVal * vals / valsMean
logging.debug("Rescaling by: " + str(normVal / valsMean))
# writing back the solutions
tw.setSelection(**coordSel)
tw.setValues(valsNew)
tw.addHistory('NORM (on axis %s)' % (normAxis))
return 0
def run( step, parset, H ):
"""
Normalize the solutions to a given value
"""
import numpy as np
from h5parm import solFetcher, solWriter
solsets = getParSolsets( step, parset, H )
soltabs = getParSoltabs( step, parset, H )
solTypes = getParSolTypes( step, parset, H )
ants = getParAxis( step, parset, H, 'ant' )
pols = getParAxis( step, parset, H, 'pol' )
dirs = getParAxis( step, parset, H, 'dir' )
normVal = parset.getFloat('.'.join(["LoSoTo.Steps", step, "NormVal"]), 1. )
normAxis = parset.getString('.'.join(["LoSoTo.Steps", step, "NormAxis"]), 'time' )
for soltab in openSoltabs( H, soltabs ):
logging.info("--> Working on soltab: "+soltab._v_name)
tr = solFetcher(soltab)
tw = solWriter(soltab)
axesNames = tr.getAxesNames()
if normAxis not in axesNames:
logging.error('Normalization axis '+normAxis+' not found.')
return 1
tr.setSelection(ant=ants, pol=pols, dir=dirs)
for vals, coord in tr.getValuesIter(returnAxes=normAxis):
# construct grid
coordSel = removeKeys(coord, normAxis)
logging.debug("Working on coords:"+str(coordSel))
# rescale solutions
valsMean = np.mean(vals)
valsNew = normVal*vals/valsMean
logging.debug("Rescaling by: "+str(normVal/valsMean))
# writing back the solutions
tw.setSelection(**coordSel)
tw.setValues(valsNew)
tw.addHistory('NORM (on axis %s)' % (normAxis))
return 0
def run(step, parset, H):
import numpy as np
from h5parm import solFetcher, solWriter
soltabs = getParSoltabs(step, parset, H)
ants = getParAxis(step, parset, H, 'ant')
pols = getParAxis(step, parset, H, 'pol')
dirs = getParAxis(step, parset, H, 'dir')
# No need to specify an axis, just use time
axesToAbs = ['time']
for soltab in openSoltabs(H, soltabs):
sf = solFetcher(soltab)
sw = solWriter(soltab)
logging.info("Taking ABSolute value of soltab: " + soltab._v_name)
sf.setSelection(ant=ants, pol=pols, dir=dirs)
total = 0
count = 0
for vals, coord in sf.getValuesIter(returnAxes=axesToAbs):
total += len(vals)
count += np.count_nonzero(vals < 0)
valsnew = np.abs(vals)
# writing back the solutions
coord = removeKeys(coord, axesToAbs)
sw.setSelection(**coord)
sw.setValues(valsnew)
logging.info('Abs: %i points initially negative (%f %%)' %
(count, float(count) / total))
sw.addHistory('ABSolute value taken')
return 0
def run( step, parset, H ):
import numpy as np
from h5parm import solFetcher, solWriter
soltabs = getParSoltabs( step, parset, H )
ants = getParAxis( step, parset, H, 'ant' )
pols = getParAxis( step, parset, H, 'pol' )
dirs = getParAxis( step, parset, H, 'dir' )
# No need to specify an axis, just use time
axesToAbs = ['time']
for soltab in openSoltabs( H, soltabs ):
sf = solFetcher(soltab)
sw = solWriter(soltab)
logging.info("Taking ABSolute value of soltab: "+soltab._v_name)
sf.setSelection(ant=ants, pol=pols, dir=dirs)
total=0
count=0
for vals, coord in sf.getValuesIter(returnAxes=axesToAbs):
total+=len(vals)
count+=np.count_nonzero(vals<0)
valsnew=np.abs(vals)
# writing back the solutions
coord = removeKeys(coord, axesToAbs)
sw.setSelection(**coord)
sw.setValues(valsnew)
logging.info('Abs: %i points initially negative (%f %%)' % (count,float(count)/total))
sw.addHistory('ABSolute value taken')
return 0
def run( step, parset, H ):
import matplotlib.pyplot as plt
import numpy as np
from h5parm import solFetcher
solsets = getParSolsets( step, parset, H )
soltabs = getParSoltabs( step, parset, H )
ants = getParAxis( step, parset, H, 'ant' )
pols = getParAxis( step, parset, H, 'pol' )
dirs = getParAxis( step, parset, H, 'dir' )
plotType = parset.getString('.'.join(["LoSoTo.Steps", step, "PlotType"]), '' )
axesToPlot = parset.getStringVector('.'.join(["LoSoTo.Steps", step, "Axes"]), '' )
minZ, maxZ = parset.getDoubleVector('.'.join(["LoSoTo.Steps", step, "MinMax"]), [0,0] )
prefix = parset.getString('.'.join(["LoSoTo.Steps", step, "Prefix"]), '' )
if plotType.lower() in ['1d', '2d']:
for soltab in openSoltabs( H, soltabs ):
sf = solFetcher(soltab)
logging.info("Plotting soltab: "+soltab._v_name)
sf.setSelection(ant=ants, pol=pols, dir=dirs)
# some checks
for axis in axesToPlot:
if axis not in sf.getAxesNames():
logging.error('Axis \"'+axis+'\" not found.')
return 1
if (len(axesToPlot) != 2 and plotType == '2D') or \
(len(axesToPlot) != 1 and plotType == '1D'):
logging.error('Wrong number of axes.')
return 1
for vals, coord in sf.getValuesIter(returnAxes=axesToPlot):
# TODO: implement flag control, using different color?
title = ''
for axis in coord:
if axis in axesToPlot: continue
title += str(coord[axis])+'_'
title = title[:-1]
if plotType == '2D':
fig = plt.figure()
ax = plt.subplot(111)
plt.title(title)
plt.ylabel(axesToPlot[0])
plt.xlabel(axesToPlot[1])
p = ax.imshow(coord[axesToPlot[1]], coord[axesToPlot[0]], vals)
if not (minZ == 0 and maxZ == 0):
plt.zlim(zmin=minZ, zmax=maxZ)
plt.savefig(title+'.png')
logging.info("Saving "+prefix+title+'.png')
if plotType == '1D':
fig = plt.figure()
ax = plt.subplot(111)
plt.title(title)
plt.ylabel(sf.getType())
if not (minZ == 0 and maxZ == 0):
plt.ylim(ymin=minZ, ymax=maxZ)
plt.xlabel(axesToPlot[0])
p = ax.plot(coord[axesToPlot[0]], vals)
plt.savefig(prefix+title+'.png')
logging.info("Saving "+prefix+title+'.png')
elif plotType.lower() == 'tecscreen':
# Plot various TEC-screen properties
for st_scr in openSoltabs(H, soltabs):
# Check if soltab is a tecscreen table
full_name = st_scr._v_parent._v_name + '/' + st_scr._v_name
if st_scr._v_title != 'tecscreen':
logging.warning('Solution table {0} is not a tecscreen solution '
'table. Skipping.'.format(full_name))
continue
logging.info('Using input solution table: {0}'.format(full_name))
# Plot TEC screens as images
solset = st_scr._v_parent
sf_scr = solFetcher(st_scr)
r, axis_vals = sf_scr.getValues()
source_names = axis_vals['dir']
station_names = axis_vals['ant']
station_dict = H.getAnt(solset)
station_positions = []
for station in station_names:
station_positions.append(station_dict[station])
times = axis_vals['time']
tec_screen, axis_vals = sf_scr.getValues()
times = axis_vals['time']
residuals = sf_scr.getValues(weight=True, retAxesVals=False)
height = st_scr._v_attrs['height']
order = st_scr._v_attrs['order']
beta_val = st_scr._v_attrs['beta']
r_0 = st_scr._v_attrs['r_0']
pp = sf_scr.t.piercepoint
if (minZ == 0 and maxZ == 0):
min_tec = None
max_tec = None
else:
min_tec = minZ
max_tec = maxZ
make_tec_screen_plots(pp, tec_screen, residuals,
np.array(station_positions), np.array(source_names), times,
height, order, beta_val, r_0, prefix=prefix,
remove_gradient=True, show_source_names=False, min_tec=min_tec,
max_tec=max_tec)
return 0
def run( step, parset, H ):
"""
Interpolate the solutions from one table into a destination table
"""
import itertools
import scipy.interpolate
import numpy as np
from h5parm import solFetcher, solWriter
solsets = getParSolsets( step, parset, H )
soltabs = getParSoltabs( step, parset, H )
solTypes = getParSolTypes( step, parset, H )
ants = getParAxis( step, parset, H, 'ant' )
pols = getParAxis( step, parset, H, 'pol' )
dirs = getParAxis( step, parset, H, 'dir' )
calSoltab = parset.getString('.'.join(["LoSoTo.Steps", step, "CalSoltab"]), '' )
calDir = parset.getString('.'.join(["LoSoTo.Steps", step, "CalDir"]), '' )
interpAxes = parset.getStringVector('.'.join(["LoSoTo.Steps", step, "InterpAxes"]), ['time','freq'] )
interpMethod = parset.getString('.'.join(["LoSoTo.Steps", step, "InterpMethod"]), 'linear' )
medAxes = parset.getStringVector('.'.join(["LoSoTo.Steps", step, "MedAxes"]), [''] )
rescale = parset.getBool('.'.join(["LoSoTo.Steps", step, "Rescale"]), False )
if interpMethod not in ["nearest", "linear", "cubic"]:
logging.error('Interpolation method must be nearest, linear or cubic.')
return 1
if rescale and medAxes == []:
logging.error('A medAxis is needed for rescaling.')
return 1
# open calibration table
css, cst = calSoltab.split('/')
cr = solFetcher(H.getSoltab(css, cst))
cAxesNames = cr.getAxesNames()
for soltab in openSoltabs( H, soltabs ):
logging.info("--> Working on soltab: "+soltab._v_name)
tr = solFetcher(soltab)
tw = solWriter(soltab)
axesNames = tr.getAxesNames()
for i, interpAxis in enumerate(interpAxes[:]):
if interpAxis not in axesNames or interpAxis not in cAxesNames:
logging.error('Axis '+interpAxis+' not found. Ignoring.')
del interpAxes[i]
for i, medAxis in enumerate(medAxes[:]):
if medAxis not in axesNames or medAxis not in cAxesNames:
logging.error('Axis '+medAxis+' not found. Ignoring.')
del medAxes[i]
tr.setSelection(ant=ants, pol=pols, dir=dirs)
for vals, coord in tr.getValuesIter(returnAxes=interpAxes):
# construct grid
coordSel = removeKeys(coord, interpAxes)
logging.debug("Working on coords:"+str(coordSel))
# change dir if sepcified
if calDir != '':
coordSel['dir'] = calDir
cr.setSelection(**coordSel)
calValues, calCoord = cr.getValues()
# fill all medAxes with the median value
for medAxis in medAxes:
axis = cAxesNames.index(medAxis)
calValues = np.repeat( np.expand_dims( np.median( calValues, axis ), axis ), calValues.shape[axis], axis )
# create a list of values whose coords are calPoints
calValues = np.ndarray.flatten(calValues)
# create calibrator/target coordinates arrays
calPoints = []
targetPoints = []
for interpAxis in interpAxes:
calPoints.append(calCoord[interpAxis])
targetPoints.append(coord[interpAxis])
calPoints = np.array([x for x in itertools.product(*calPoints)])
targetPoints = np.array([x for x in itertools.product(*targetPoints)])
# interpolation
valsNew = scipy.interpolate.griddata(calPoints, calValues, targetPoints, interpMethod)
# fill values outside boudaries with "nearest" solutions
if interpMethod != 'nearest':
valsNewNearest = scipy.interpolate.griddata(calPoints, calValues, targetPoints, 'nearest')
valsNew[ np.where(valsNew == np.nan) ] = valsNewNearest [ np.where(valsNew == np.nan) ]
valsNew = valsNew.reshape(vals.shape)
if rescale:
# rescale solutions
for medAxis in medAxes:
axis = interpAxes.index(medAxis)
valsMed = np.repeat( np.expand_dims( np.median( vals, axis ), axis ), vals.shape[axis], axis )
valsNewMed = np.repeat( np.expand_dims( np.median( valsNew, axis ), axis ), valsNew.shape[axis], axis )
valsNew = vals*valsNewMed/valsMed
#print "Rescaling by: ", valsNewMed[:,0]/valsMed[:,0]
# writing back the solutions
coord = removeKeys(coord, interpAxes)
tw.setSelection(**coord)
tw.setValues(valsNew)
tw.addHistory('INTERP (from table %s)' % (calSoltab))
return 0
def collect_solutions(H, dirs=None, freq_tol=1e6, solsets=None):
"""
Collects and returns phase solutions, etc. needed for fitting
Keyword arguments:
H -- H5parm object
dirs -- list of directions to use (None => all)
freq_tol -- tolerance for grouping of phase solutions into bands (Hz)
solsets -- list of solution sets in H to search (None => all)
"""
import numpy as np
from pylab import find
import progressbar
import re
logging.info("Scanning for solutions needed for TEC fitting...")
# Determine axis lengths
sources = []
freqs = []
stations = []
if solsets is None:
solsets = H.getSolsets().keys()
N_times_max = 0
first_solset = None
for solset in solsets[:]:
logging.info(' --Scanning solution set {0}...'.format(solset))
has_phase_st = False
soltabs = H.getSoltabs(solset)
for soltab in soltabs:
# Only use soltabs with phase solutions
if 'phase' in soltabs[soltab]._v_title:
logging.info(
' --Scanning solution table {0}...'.format(soltab))
has_phase_st = True
solset_sources = H.getSou(solset)
if len(solset_sources) == 1 and 'pointing' in solset_sources:
logging.info(' Found direction-independent solutions')
dir_indep = True
soln_type_dirindep = soltabs[soltab]._v_title
if first_solset is None:
first_solset = solset
else:
logging.info(' Found direction-dependent solutions')
dir_indep = False
if first_solset is None:
first_solset = solset
if not dir_indep:
soln_type_dirdep = soltabs[soltab]._v_title
logging.info(' Found sources: {0}'.format(
H.getSou(solset).keys()))
sources += H.getSou(solset).keys()
stations += H.getAnt(solset).keys()
t = solFetcher(soltabs[soltab])
solns, axis_vals = t.getValues()
times = axis_vals['time']
N_times = len(times)
freqs.append(axis_vals['freq'][0])
if N_times > N_times_max:
N_times_max = N_times
times_max = times
if not has_phase_st:
logging.info(' --No phase solutions found')
solsets.remove(solset)
# Consolidate frequencies into bands (defined by freq_tol parameter)
has_dup = True
while has_dup:
has_dup = False
for freq in freqs[:]:
nfreq = len(
find((np.array(freqs) > (freq - freq_tol))
& (np.array(freqs) < (freq + freq_tol))))
if nfreq > 1:
has_dup = True
freqs.remove(freq)
break
freqs = np.array(sorted(freqs))
N_freqs = len(freqs)
stations_set = set(stations)
sources_set = set(sources)
if 'pointing' in sources_set:
sources_set.remove('pointing')
if dirs is not None:
sources_filt = []
for dir in dirs:
if dir in sources_set:
sources_filt.append(dir)
sources_set = set(sources_filt)
source_names = np.array(sorted(list(sources_set)))
N_sources = len(set(source_names))
N_stations = len(set(stations))
N_times = N_times_max
logging.info('Scanning complete')
logging.info(' Number of sources: {0}'.format(N_sources))
logging.info(' Number of stations: {0}'.format(N_stations))
logging.info(' Number of times: {0}'.format(N_times))
logging.info(' Number of bands: {0}'.format(N_freqs))
if N_sources == 0 or N_stations == 0 or N_times == 0 or N_freqs == 0:
logging.error('No solutions found.')
return (None, None, None, None, None, None, None, None, None, None,
None, None)
# Initialize the arrays
freqwidths = np.zeros(N_freqs)
timewidths = np.zeros(N_times)
m = np.zeros((N_sources, N_freqs))
phases0 = np.zeros((N_sources, N_stations, N_freqs, N_times))
phases1 = np.zeros((N_sources, N_stations, N_freqs, N_times))
flags = np.ones((N_sources, N_stations, N_freqs, N_times))
source_positions = np.zeros((N_sources, 2))
solset_array_dir_dep = np.zeros((N_sources, N_freqs), dtype='|S100')
solset_array_dir_indep = np.zeros(N_freqs, dtype='|S100')
# Populate the arrays
for solset in solsets:
source_dict = H.getSou(solset)
sources = source_dict.keys()
soltabs = H.getSoltabs(solset)
for soltab in soltabs:
t = solFetcher(soltabs[soltab])
solns, axes = t.getValues()
freq = axes['freq'][0]
freq_indx = find((np.array(freqs) > (freq - freq_tol))
& (np.array(freqs) < (freq + freq_tol)))
for source in sources:
if source in source_names:
source_indx = find(source_names == source)
m[source_indx, freq_indx] = 1
solset_array_dir_dep[source_indx, freq_indx] = solset
elif source == 'pointing' and len(sources) == 1:
solset_array_dir_indep[freq_indx] = solset
# Collect station properties and pointing direction
station_dict = H.getAnt(first_solset)
station_names = np.array(list(stations_set))
station_positions = np.zeros((len(station_names), 3), dtype=np.float)
for i, station_name in enumerate(station_names):
station_positions[i, 0] = station_dict[station_name][0]
station_positions[i, 1] = station_dict[station_name][1]
station_positions[i, 2] = station_dict[station_name][2]
source_dict = H.getSou(first_solset)
ra = source_dict['pointing'][0]
dec = source_dict['pointing'][1]
pointing = np.array(
[np.cos(ra) * np.cos(dec),
np.sin(ra) * np.cos(dec),
np.sin(dec)])
# Collect source positions and phase solutions for each band
logging.info('Collecting phase solutions...')
pbar = progressbar.ProgressBar(maxval=len(source_names)).start()
for i, source1 in enumerate(source_names):
for k in range(N_freqs):
if m[i, k]:
solset_name = str(solset_array_dir_dep[i, k])
soltab = H.getSoltab(solset=solset_name,
soltab=soln_type_dirdep + '000')
solset_dir_indep_name = str(solset_array_dir_indep[k])
source_dict = H.getSou(solset_name)
source_positions[i, ] = source_dict[source1]
if solset_dir_indep_name != '':
soltab_dir_indep = H.getSoltab(
solset=solset_dir_indep_name,
soltab=soln_type_dirindep + '000')
sf_dir_indep = solFetcher(soltab_dir_indep)
else:
sf_dir_indep = None
sf_dir_dep = solFetcher(soltab)
for l, station in enumerate(station_names):
if soln_type_dirdep == 'scalarphase':
sf_dir_dep.setSelection(ant=station, dir=source1)
else:
sf_dir_dep.setSelection(ant=station,
pol='XX',
dir=source1)
values_dir_dep = sf_dir_dep.getValues()
v1_phase = np.array(values_dir_dep[0]).squeeze()
times_dir_dep = values_dir_dep[1]['time']
if sf_dir_indep is not None:
if soln_type_dirindep == 'scalarphase':
sf_dir_indep.setSelection(ant=station)
else:
sf_dir_indep.setSelection(ant=station, pol='XX')
values_dir_indep = sf_dir_indep.getValues()
v1_dir_indep = np.array(values_dir_indep[0]).squeeze()
times_dir_indep = values_dir_indep[1]['time']
v1_dir_indep_interp = interpolate_phase(
v1_dir_indep, times_dir_indep, times_dir_dep)
v1_phase += v1_dir_indep_interp
if len(times_dir_dep) != N_times_max:
phases0 = interpolate_phase(phases0, times_dir_indep,
times_max)
phases0[i, l, k, :] = v1_phase
if soln_type_dirdep == 'scalarphase':
phases1[i, l, k, :] = v1_phase
else:
sf_dir_dep.setSelection(ant=station,
pol='YY',
dir=source1)
values_dir_dep = sf_dir_dep.getValues()
v1_phase = np.array(values_dir_dep[0]).squeeze()
times_dir_dep = values_dir_dep[1]['time']
if sf_dir_indep is not None:
if soln_type_dirindep == 'scalarphase':
sf_dir_indep.setSelection(ant=station)
else:
sf_dir_indep.setSelection(ant=station,
pol='YY')
values_dir_indep = sf_dir_indep.getValues()
v1_dir_indep = np.array(
values_dir_indep[0]).squeeze()
times_dir_indep = values_dir_indep[1]['time']
v1_dir_indep_interp = interpolate_phase(
v1_dir_indep, times_dir_indep, times_dir_dep)
v1_phase += v1_dir_indep_interp
if len(times_dir_dep) != N_times_max:
phases1 = interpolate_phase(
phases1, times_dir_indep, times_max)
phases1[i, l, k, :] = v1_phase
flags[i, l, k, :] = sf_dir_dep.getValues(weight=True,
retAxesVals=False)
if np.all(phases0[i, l, k, :] == 0.0):
# Check for flagged stations
flags[i, l, k, :] = 0.0
pbar.update(i)
pbar.finish()
logging.info('Collection complete')
for i, source in enumerate(source_names):
nbands = len(find(m[i, :]))
logging.info(' Source {0} has solutions in {1} bands'.format(
source, nbands))
# Invert the weights to give flags (0 => unflagged, 1 => flagged)
zeroflags = np.where(flags == 0.0)
oneflags = flags.nonzero()
flags[zeroflags] = 1.0
flags[oneflags] = 0.0
return (phases0, phases1, flags, m, station_names, station_positions,
source_names, source_positions, freqs, times, pointing,
soln_type_dirdep)
def run(step, parset, H):
import numpy as np
from h5parm import solFetcher, solWriter
soltabs = getParSoltabs(step, parset, H)
ants = getParAxis(step, parset, H, 'ant')
pols = getParAxis(step, parset, H, 'pol')
dirs = getParAxis(step, parset, H, 'dir')
axesToClip = parset.getStringVector(
'.'.join(["LoSoTo.Steps", step, "Axes"]), [])
clipLevel = parset.getFloat('.'.join(["LoSoTo.Steps", step, "ClipLevel"]),
0.)
if len(axesToClip) < 1:
logging.error("Please specify axes to clip.")
return 1
if clipLevel == 0.:
logging.error(
"Please specify factor above/below median at which to clip.")
return 1
for soltab in openSoltabs(H, soltabs):
sf = solFetcher(soltab)
sw = solWriter(soltab)
logging.info("Clipping soltab: " + soltab._v_name)
sf.setSelection(ant=ants, pol=pols, dir=dirs)
# some checks
for i, axis in enumerate(axesToClip[:]):
if axis not in sf.getAxesNames():
del axesToClip[i]
logging.warning('Axis \"' + axis + '\" not found. Ignoring.')
before_count = 0
after_count = 0
total = 0
for vals, coord in sf.getValuesIter(returnAxes=axesToClip):
total += len(vals)
before_count += np.count_nonzero(np.isnan(vals))
# clipping
# first find the median and standard deviation
valmedian = np.median(vals)
clipvalue = valmedian * clipLevel
np.putmask(vals, vals > clipvalue, np.nan)
clipvalue = valmedian / clipLevel
np.putmask(vals, vals < clipvalue, np.nan)
after_count += np.count_nonzero(np.isnan(vals))
# writing back the solutions
coord = removeKeys(coord, axesToClip)
sw.setSelection(**coord)
sw.setValues(vals)
sw.addHistory('CLIP (over %s with %s sigma cut)' %
(axesToClip, clipLevel))
logging.info(
'Clip: %i points initially bad, %i after clipping (%f %%)' %
(before_count, after_count, float(after_count) / total))
return 0
def run( step, parset, H ):
import numpy as np
from h5parm import solFetcher, solWriter
soltabs = getParSoltabs( step, parset, H )
ants = getParAxis( step, parset, H, 'ant' )
pols = getParAxis( step, parset, H, 'pol' )
dirs = getParAxis( step, parset, H, 'dir' )
axesToMed = parset.getStringVector('.'.join(["LoSoTo.Steps", step, "Axes"]), [] )
clipLevel = parset.getFloat('.'.join(["LoSoTo.Steps", step, "SigLevel"]), 0. )
if len(axesToMed) < 1:
logging.error("Please specify axes.")
return 1
if clipLevel == 0.:
logging.error("Please specify significance level above which to clip.")
return 1
for soltab in openSoltabs( H, soltabs ):
sf = solFetcher(soltab)
sw = solWriter(soltab)
logging.info("Flagging soltab: "+soltab._v_name)
sf.setSelection(ant=ants, pol=pols, dir=dirs)
# some checks
if len(axesToMed) < 1:
logging.error("Please specify axes.")
return 1
if clipLevel == 0.:
logging.error("Please specify significance level above which to clip.")
return 1
for i, axis in enumerate(axesToMed[:]):
if axis not in sf.getAxesNames():
del axesToMed[i]
logging.warning('Axis \"'+axis+'\" not found. Ignoring.')
before_count=0
after_count=0
total=0
for vals, coord in sf.getValuesIter(returnAxes=axesToMed):
total+=len(vals)
before_count+=np.count_nonzero(np.isnan(vals))
# clipping
# first find the median and standard deviation
valmedian = np.median(vals)
clipvalue = valmedian * clipLevel
np.putmask(vals, vals > clipvalue, np.nan)
clipvalue = valmedian / clipLevel
np.putmask(vals, vals < clipvalue, np.nan)
after_count+=np.count_nonzero(np.isnan(vals))
# writing back the solutions
coord = removeKeys(coord, axesToMed)
sw.setSelection(**coord)
sw.setValues(vals)
sw.addHistory('FLAG (over %s with %s sigma cut)' % (axesToMed, clipLevel))
logging.info('Clip: %i points initially bad, %i after flagging (%f %%)' % (before_count,after_count,float(after_count)/total))
return 0
def run(step, parset, H):
import matplotlib.pyplot as plt
import numpy as np
from h5parm import solFetcher
solsets = getParSolsets(step, parset, H)
soltabs = getParSoltabs(step, parset, H)
ants = getParAxis(step, parset, H, 'ant')
pols = getParAxis(step, parset, H, 'pol')
dirs = getParAxis(step, parset, H, 'dir')
plotType = parset.getString('.'.join(["LoSoTo.Steps", step, "PlotType"]),
'')
axesToPlot = parset.getStringVector(
'.'.join(["LoSoTo.Steps", step, "Axes"]), '')
minZ, maxZ = parset.getDoubleVector(
'.'.join(["LoSoTo.Steps", step, "MinMax"]), [0, 0])
prefix = parset.getString('.'.join(["LoSoTo.Steps", step, "Prefix"]), '')
if plotType.lower() in ['1d', '2d']:
for soltab in openSoltabs(H, soltabs):
sf = solFetcher(soltab)
logging.info("Plotting soltab: " + soltab._v_name)
sf.setSelection(ant=ants, pol=pols, dir=dirs)
# some checks
for axis in axesToPlot:
if axis not in sf.getAxesNames():
logging.error('Axis \"' + axis + '\" not found.')
return 1
if (len(axesToPlot) != 2 and plotType == '2D') or \
(len(axesToPlot) != 1 and plotType == '1D'):
logging.error('Wrong number of axes.')
return 1
for vals, coord in sf.getValuesIter(returnAxes=axesToPlot):
# TODO: implement flag control, using different color?
title = ''
for axis in coord:
if axis in axesToPlot: continue
title += str(coord[axis]) + '_'
title = title[:-1]
if plotType == '2D':
fig = plt.figure()
ax = plt.subplot(111)
plt.title(title)
plt.ylabel(axesToPlot[0])
plt.xlabel(axesToPlot[1])
p = ax.imshow(coord[axesToPlot[1]], coord[axesToPlot[0]],
vals)
if not (minZ == 0 and maxZ == 0):
plt.zlim(zmin=minZ, zmax=maxZ)
plt.savefig(title + '.png')
logging.info("Saving " + prefix + title + '.png')
if plotType == '1D':
fig = plt.figure()
ax = plt.subplot(111)
plt.title(title)
plt.ylabel(sf.getType())
if not (minZ == 0 and maxZ == 0):
plt.ylim(ymin=minZ, ymax=maxZ)
plt.xlabel(axesToPlot[0])
p = ax.plot(coord[axesToPlot[0]], vals)
plt.savefig(prefix + title + '.png')
logging.info("Saving " + prefix + title + '.png')
elif plotType.lower() == 'tecscreen':
# Plot various TEC-screen properties
for st_scr in openSoltabs(H, soltabs):
# Check if soltab is a tecscreen table
full_name = st_scr._v_parent._v_name + '/' + st_scr._v_name
if st_scr._v_title != 'tecscreen':
logging.warning(
'Solution table {0} is not a tecscreen solution '
'table. Skipping.'.format(full_name))
continue
logging.info('Using input solution table: {0}'.format(full_name))
# Plot TEC screens as images
solset = st_scr._v_parent
sf_scr = solFetcher(st_scr)
r, axis_vals = sf_scr.getValues()
source_names = axis_vals['dir']
station_names = axis_vals['ant']
station_dict = H.getAnt(solset)
station_positions = []
for station in station_names:
station_positions.append(station_dict[station])
times = axis_vals['time']
tec_screen, axis_vals = sf_scr.getValues()
times = axis_vals['time']
residuals = sf_scr.getValues(weight=True, retAxesVals=False)
height = st_scr._v_attrs['height']
order = st_scr._v_attrs['order']
beta_val = st_scr._v_attrs['beta']
r_0 = st_scr._v_attrs['r_0']
pp = sf_scr.t.piercepoint
if (minZ == 0 and maxZ == 0):
min_tec = None
max_tec = None
else:
min_tec = minZ
max_tec = maxZ
make_tec_screen_plots(pp,
tec_screen,
residuals,
np.array(station_positions),
np.array(source_names),
times,
height,
order,
beta_val,
r_0,
prefix=prefix,
remove_gradient=True,
show_source_names=False,
min_tec=min_tec,
max_tec=max_tec)
return 0
def run( step, parset, H ):
"""
Generic unspecified step for easy expansion.
"""
import numpy as np
from h5parm import solFetcher, solWriter
# all the following are LoSoTo function to extract information from the parset
# get involved solsets using local step values or global values or all
solsets = getParSolsets( step, parset, H )
logging.info('Solset: '+str(solsets))
# get involved soltabs using local step values or global values or all
soltabs = getParSoltabs( step, parset, H )
logging.info('Soltab: '+str(soltabs))
# get list of Antennas using local step values or global values or all
ants = getParAxis( step, parset, H, 'ant' )
logging.info('Ant: '+str(ants))
# get list of Polarizations using local step values or global values or all
pols = getParAxis( step, parset, H, 'pol' )
logging.info('Pol: '+str(pols))
# get list of SolTypes using local step values or global values or all
solTypes = getParSolTypes( step, parset, H )
logging.info('SolType: '+str(solTypes))
# get list of Directions using local step values or global values or all
dirs = getParAxis( step, parset, H, 'dir' )
logging.info('Dir: '+str(dirs))
# do something on every soltab (use the openSoltab LoSoTo function)
for soltab in openSoltabs( H, soltabs ):
logging.info("--> Working on soltab: "+soltab._v_name)
# use the solFetcher from the H5parm lib
t = solFetcher(soltab)
tw = solWriter(soltab)
axisNames = t.getAxesNames()
logging.info("Axis names are: "+str(axisNames))
solType = t.getType()
logging.info("Soltab type is: "+solType)
# this will make a selection for the getValues() and getValuesIter()
t.setSelection(ant=ants, pol=pols, dir=dirs)
logging.info("Selection is: "+str(t.selection))
# find axis values
logging.info("Antennas (no selection) are: "+str(t.getAxisValues('ant', ignoreSelection=True)))
logging.info("Antennas (with selection) are: "+str(t.getAxisValues('ant')))
# but one can also use (selection is active here!)
logging.info("Antennas (other method) are: "+str(t.ant))
logging.info("Frequencies are: "+str(t.freq))
logging.info("Directions are: "+str(t.dir))
logging.info("Polarizations are: "+str(t.pol))
# try to access a non-existent axis
t.getAxisValues('nonexistantaxis')
# now get all values given this selection
logging.info("Get data using t.val")
val = t.val
logging.debug('shape of val: '+str(t.val.shape))
logging.info("$ val is "+str(val[0,0,0,0,100]))
weight = t.weight
time = t.time
thisTime = t.time[100]
# another way to get the data is using the getValues()
logging.info("Get data using getValues()")
grid, axes = t.getValues()
# axis names
logging.info("Axes: "+str(t.getAxesNames()))
# axis shape
print axes
print [t.getAxisLen(axis) for axis in axes] # not ordered, is a dict!
# data array shape (same of axis shape)
logging.info("Shape of values: "+str(grid.shape))
#logging.info("$ val is "+str(grid[0,0,0,0,100]))
# reset selection
t.setSelection()
logging.info('Reset selection to \'\'')
logging.info("Antennas are: "+str(t.ant))
logging.info("Frequencies are: "+str(t.freq))
logging.info("Directions are: "+str(t.dir))
logging.info("Polarizations are: "+str(t.pol))
# finally the getValuesIter allaws to iterate across all possible combinations of a set of axes
logging.info('Iteration on time/freq')
for vals, coord in t.getValuesIter(returnAxes=['time','freq']):
# writing back the solutions
coord = removeKeys(coord, ['time','freq']) # removeKeys is an function of the operation_lib
tw.setSelection(**coord)
tw.setValues(vals)
logging.info('Iteration on time')
for vals, coord in t.getValuesIter(returnAxes=['time']):
# writing back the solutions
coord = removeKeys(coord, ['time','freq']) # removeKeys is an function of the operation_lib
tw.setSelection(**coord)
tw.setValues(vals)
logging.info('Iteration on dir after selection to 1 dir')
t.setSelection(dir='pointing')
for vals, coord in t.getValuesIter(returnAxes=['dir']):
# writing back the solutions
coord = removeKeys(coord, ['time','freq']) # removeKeys is an function of the operation_lib
tw.setSelection(**coord)
tw.setValues(vals)
return 0 # if everything went fine, otherwise 1
def run( step, parset, H ):
import matplotlib.pyplot as plt
import numpy as np
from h5parm import solFetcher
solsets = getParSolsets( step, parset, H )
soltabs = getParSoltabs( step, parset, H )
ants = getParAxis( step, parset, H, 'ant' )
pols = getParAxis( step, parset, H, 'pol' )
dirs = getParAxis( step, parset, H, 'dir' )
plotType = parset.getString('.'.join(["LoSoTo.Steps", step, "PlotType"]), '' )
axesToPlot = parset.getStringVector('.'.join(["LoSoTo.Steps", step, "Axes"]), '' )
minZ, maxZ = parset.getDoubleVector('.'.join(["LoSoTo.Steps", step, "MinMax"]), [0,0] )
prefix = parset.getString('.'.join(["LoSoTo.Steps", step, "Prefix"]), '' )
if plotType.lower() != 'tecscreen':
for soltab in openSoltabs( H, soltabs ):
sf = solFetcher(soltab)
logging.info("Plotting soltab: "+soltab._v_name)
sf.setSelection(ant=ants, pol=pols, dir=dirs)
# some checks
for axis in axesToPlot:
if axis not in sf.getAxesNames():
logging.error('Axis \"'+axis+'\" not found.')
return 1
if (len(axesToPlot) != 2 and plotType == '2D') or \
(len(axesToPlot) != 1 and plotType == '1D'):
logging.error('Wrong number of axes.')
return 1
for vals, coord in sf.getValuesIter(returnAxes=axesToPlot):
# TODO: implement flag control, using different color?
title = ''
for axis in coord:
if axis in axesToPlot: continue
title += str(coord[axis])+'_'
title = title[:-1]
if plotType == '2D':
fig = plt.figure()
ax = plt.subplot(111)
plt.title(title)
plt.ylabel(axesToPlot[0])
plt.xlabel(axesToPlot[1])
p = ax.imshow(coord[axesToPlot[1]], coord[axesToPlot[0]], vals)
if not (minZ == 0 and maxZ == 0):
plt.zlim(zmin=minZ, zmax=maxZ)
plt.savefig(title+'.png')
logging.info("Saving "+prefix+title+'.png')
if plotType == '1D':
fig = plt.figure()
ax = plt.subplot(111)
plt.title(title)
plt.ylabel(sf.getType())
if not (minZ == 0 and maxZ == 0):
plt.ylim(ymin=minZ, ymax=maxZ)
plt.xlabel(axesToPlot[0])
p = ax.plot(coord[axesToPlot[0]], vals)
plt.savefig(prefix+title+'.png')
logging.info("Saving "+prefix+title+'.png')
else:
# Plot TEC screens
i = 0
st_tec = None
st_pp = None
st_tfw = None
# Find required soltabs
for st in openSoltabs(H, soltabs):
if st._v_title == 'tec':
st_tec = st
tec_indx = i
elif st._v_title == 'tecfitwhite':
st_tfw = st
elif st._v_title == 'piercepoint':
st_pp = st
i += 1
if st_tec is None or st_pp is None or st_tfw is None:
logging.warning('One or more of the required TEC solution tables '
'not found')
return 1
solset = soltabs[tec_indx].split('/')[0]
station_dict = H.getAnt(solset)
station_names = station_dict.keys()
station_positions = station_dict.values()
source_dict = H.getSou(solset)
source_names = source_dict.keys()
source_positions = source_dict.values()
sf_tec = solFetcher(st_tec)
r, axis_vals = sf_tec.getValues()
times = axis_vals['time']
N_sources = len(source_names)
N_times = len(times)
N_stations = len(station_names)
N_piercepoints = N_sources * N_stations
rr = np.reshape(r.transpose([0, 2, 1]), [ N_piercepoints, N_times])
sf_pp = solFetcher(st_pp)
pp = sf_pp.getValues(retAxesVals=False)
height = st_tfw._v_attrs['height']
order = st_tfw._v_attrs['order']
beta_val = st_tfw._v_attrs['beta']
r_0 = st_tfw._v_attrs['r_0']
sf_tfw = solFetcher(st_tfw)
tec_fit_white = sf_tfw.getValues(retAxesVals=False)
tec_fit = sf_tfw.getValues(weight=True, retAxesVals=False)
make_tec_screen_plots(pp, rr, tec_fit_white, tec_fit,
np.array(station_positions), np.array(source_names), times,
height, order, beta_val, r_0, prefix=prefix, remove_gradient=True)
return 0
def run(step, parset, H):
import numpy as np
from h5parm import solFetcher, solWriter
soltabs = getParSoltabs(step, parset, H)
ants = getParAxis(step, parset, H, 'ant')
pols = getParAxis(step, parset, H, 'pol')
dirs = getParAxis(step, parset, H, 'dir')
axesToMed = parset.getStringVector(
'.'.join(["LoSoTo.Steps", step, "Axes"]), [])
clipLevel = parset.getFloat('.'.join(["LoSoTo.Steps", step, "SigLevel"]),
0.)
if len(axesToMed) < 1:
logging.error("Please specify axes.")
return 1
if clipLevel == 0.:
logging.error("Please specify significance level above which to clip.")
return 1
for soltab in openSoltabs(H, soltabs):
sf = solFetcher(soltab)
sw = solWriter(soltab)
logging.info("Smoothing soltab: " + soltab.name)
sf.setSelection(ant=ants, pol=pols, dir=dirs)
# some checks
if len(axesToMed) < 1:
logging.error("Please specify axes.")
return 1
if clipLevel == 0.:
logging.error(
"Please specify significance level above which to clip.")
return 1
for i, axis in enumerate(axesToMed[:]):
if axis not in sf.getAxes():
del axesToSmooth[i]
logging.warning('Axis \"' + axis + '\" not found. Ignoring.')
for vals, coord in sf.getValuesIter(returnAxes=axesToMed):
# clipping
# first find the median and standard deviation
valmedian = np.median(vals)
clipvalue = valmedian * clipLevel
np.putmask(vals, vals > clipvalue, np.nan)
# writing back the solutions
coord = removeKeys(coord, axesToSmooth)
sw.setSelection(**coord)
sw.setValues(vals)
sw.addHistory('FLAG (over %s with %s sigma cut)' %
(axesToMed, clipLevel))
return 0
def run(step, parset, H):
"""
Fits a screen to TEC values derived by the TECFIT operation.
The TEC values are read from the specified tec soltab.
The results of the fit are stored in the specified tecscreen solution table.
These values are the screen TEC values per station per pierce point per
solution interval. The pierce point locations are stored in an auxiliary
array in the output solution table.
TEC screens can be plotted with the PLOT operation by setting PlotType =
TECScreen.
The H5parm_exporter.py tool can be used to export the screen to a parmdb
that BBS and the AWimager can use. Note, however, that the output screens
are not normalized properly (any normalization was lost due to the use of
source-to-source phase gradients in the TECFIT operation). Therefore, a
direction-independent calibration must be done after exporting the screens
to a parmdb file, with the following settings in the BBS solve step:
Model.Ionosphere.Enable = T
Model.Ionosphere.Type = EXPION
"""
import numpy as np
import re
from h5parm import solFetcher, solWriter
soltabs = getParSoltabs(step, parset, H)
outSoltabs = parset.getStringVector(
'.'.join(["LoSoTo.Steps", step, "OutSoltab"]), [])
height = np.array(
parset.getDoubleVector('.'.join(["LoSoTo.Steps", step, "Height"]),
[200e3]))
order = int(
parset.getString('.'.join(["LoSoTo.Steps", step, "Order"]), '15'))
# Load TEC values from TECFIT operation
indx = 0
for soltab in openSoltabs(H, soltabs):
if 'tec' not in soltab._v_title:
logging.warning(
'No TECFIT solution tables found for solution table '
'{0}'.format(soltabs[indx]))
continue
indx += 1
solset = soltabs[indx].split('/')[0]
logging.info('Using input solution table: {0}'.format(soltabs[indx]))
logging.info('Using output solution table: {0}'.format(
outSoltabs[indx]))
# Collect station and source names and positions and times, making sure
# that they are ordered correctly.
t = solFetcher(soltab)
r, axis_vals = t.getValues()
source_names = axis_vals['dir']
source_dict = H.getSou(solset)
source_positions = []
for source in source_names:
source_positions.append(source_dict[source])
station_names = axis_vals['ant']
station_dict = H.getAnt(solset)
station_positions = []
for station in station_names:
station_positions.append(station_dict[station])
times = axis_vals['time']
# Get sizes
N_sources = len(source_names)
N_times = len(times)
N_stations = len(station_names)
N_piercepoints = N_sources * N_stations
rr = np.reshape(r.transpose([0, 2, 1]), [N_piercepoints, N_times])
heights = set(np.linspace(height[0], height[-1], 5))
if len(heights) > 1:
logging.info('Trying range of heights: {0} m'.format(
list(heights)))
for i, height in enumerate(heights):
# Find pierce points and airmass values for given screen height
logging.info('Using height = {0} m and order = {1}'.format(
height, order))
pp, airmass = calculate_piercepoints(np.array(station_positions),
np.array(source_positions),
np.array(times), height)
# Fit a TEC screen
r_0 = 10e3
beta = 5.0 / 3.0
tec_screen, residual = fit_screen_to_tec(station_names,
source_names, pp, airmass,
rr, times, height, order,
r_0, beta)
total_resid = np.sum(np.abs(residual))
if i > 0:
if total_resid < best_resid:
tec_screen_best = tec_screen
pp_best = pp
height_best = height
best_resid = total_resid
else:
tec_screen_best = tec_screen
pp_best = pp
height_best = height
best_resid = total_resid
if len(heights) > 1:
logging.info(
'Total residual for fit: {0}\n'.format(total_resid))
# Use screen with lowest total residual
if len(heights) > 1:
tec_screen = tec_screen_best
pp = pp_best
height = height_best
logging.info(
'Using height (with lowest total residual) of {0} m'.format(
height))
# Write the results to the output solset
dirs_out = source_names
times_out = times
ants_out = station_names
# Make output tecscreen table
outSolset = outSoltabs[indx].split('/')[0]
outSoltab = outSoltabs[indx].split('/')[1]
if not outSolset in H.getSolsets().keys():
solsetTEC = H.makeSolset(outSolset)
dirs_pos = source_positions
sourceTable = solsetTEC._f_get_child('source')
sourceTable.append(zip(*(dirs_out, dirs_pos)))
ants_pos = station_positions
antennaTable = solsetTEC._f_get_child('antenna')
antennaTable.append(zip(*(ants_out, ants_pos)))
# Store tecscreen values. The residual values are stored in the weights
# table
vals = tec_screen.transpose([1, 0, 2])
weights = residual.transpose([1, 0, 2])
tec_screen_st = H.makeSoltab(outSolset,
'tecscreen',
outSoltab,
axesNames=['dir', 'time', 'ant'],
axesVals=[dirs_out, times_out, ants_out],
vals=vals,
weights=weights)
# Store beta, r_0, height, and order as attributes of the tecscreen
# soltab
tec_screen_st._v_attrs['beta'] = beta
tec_screen_st._v_attrs['r_0'] = r_0
tec_screen_st._v_attrs['height'] = height
tec_screen_st._v_attrs['order'] = order
# Make output piercepoint table
tec_screen_solset = tec_screen_st._v_parent._v_name
H.H.create_carray('/' + tec_screen_solset + '/' +
tec_screen_st._v_name,
'piercepoint',
obj=pp)
# Add histories
sw = solWriter(tec_screen_st)
sw.addHistory('CREATE (by TECSCREEN operation)')
indx += 1
return 0
def run( step, parset, H ):
"""
Fits a screen to TEC values derived by the TECFIT operation.
The TEC values are read from the specified tec soltab.
The results of the fit are stored in the specified tecscreen solution table.
These values are the screen TEC values per station per pierce point per
solution interval. The pierce point locations are stored in an auxiliary
array in the output solution table.
TEC screens can be plotted with the PLOT operation by setting PlotType =
TECScreen.
The H5parm_exporter.py tool can be used to export the screen to a parmdb
that BBS and the AWimager can use. Note, however, that the output screens
are not normalized properly (any normalization was lost due to the use of
source-to-source phase gradients in the TECFIT operation). Therefore, a
direction-independent calibration must be done after exporting the screens
to a parmdb file, with the following settings in the BBS solve step:
Model.Ionosphere.Enable = T
Model.Ionosphere.Type = EXPION
"""
import numpy as np
import re
from h5parm import solFetcher, solWriter
# Switch to the Agg backend to prevent problems with pylab imports when
# DISPLAY env. variable is not set
import os
if 'DISPLAY' not in os.environ:
import matplotlib
matplotlib.use("Agg")
soltabs = getParSoltabs( step, parset, H )
outSoltabs = parset.getStringVector('.'.join(["LoSoTo.Steps", step, "OutSoltab"]), [] )
height = np.array(parset.getDoubleVector('.'.join(["LoSoTo.Steps", step, "Height"]), [200e3] ))
order = int(parset.getString('.'.join(["LoSoTo.Steps", step, "Order"]), '15' ))
# Load TEC values from TECFIT operation
indx = 0
for soltab in openSoltabs(H, soltabs):
if 'tec' not in soltab._v_title:
logging.warning('No TECFIT solution tables found for solution table '
'{0}'.format(soltabs[indx]))
continue
indx += 1
solset = soltabs[indx].split('/')[0]
logging.info('Using input solution table: {0}'.format(soltabs[indx]))
logging.info('Using output solution table: {0}'.format(outSoltabs[indx]))
# Collect station and source names and positions and times, making sure
# that they are ordered correctly.
t = solFetcher(soltab)
r, axis_vals = t.getValues()
source_names = axis_vals['dir']
source_dict = H.getSou(solset)
source_positions = []
for source in source_names:
source_positions.append(source_dict[source])
station_names = axis_vals['ant']
station_dict = H.getAnt(solset)
station_positions = []
for station in station_names:
station_positions.append(station_dict[station])
times = axis_vals['time']
# Get sizes
N_sources = len(source_names)
N_times = len(times)
N_stations = len(station_names)
N_piercepoints = N_sources * N_stations
rr = np.reshape(r.transpose([0, 2, 1]), [N_piercepoints, N_times])
heights = list(set(np.linspace(height[0], height[-1], 5)))
heights.sort()
if len(heights) > 1:
logging.info('Trying range of heights: {0} m'.format(heights))
for i, height in enumerate(heights):
# Find pierce points and airmass values for given screen height
logging.info('Using height = {0} m and order = {1}'.format(height, order))
if height < 100e3:
logging.warning("Height is less than 100e3 m.")
pp, airmass = calculate_piercepoints(np.array(station_positions),
np.array(source_positions), np.array(times), height)
# Fit a TEC screen
r_0 = 10e3
beta = 5.0 / 3.0
tec_screen, residual = fit_screen_to_tec(station_names, source_names,
pp, airmass, rr, times, height, order, r_0, beta)
total_resid = np.sum(np.abs(residual))
if i > 0:
if total_resid < best_resid:
tec_screen_best = tec_screen
pp_best = pp
height_best = height
best_resid = total_resid
else:
tec_screen_best = tec_screen
pp_best = pp
height_best = height
best_resid = total_resid
if len(heights) > 1:
logging.info('Total residual for fit: {0}\n'.format(total_resid))
# Use screen with lowest total residual
if len(heights) > 1:
tec_screen = tec_screen_best
pp = pp_best
height = height_best
logging.info('Using height (with lowest total residual) of {0} m'.format(height))
# Write the results to the output solset
dirs_out = source_names
times_out = times
ants_out = station_names
# Make output tecscreen table
outSolset = outSoltabs[indx].split('/')[0]
outSoltab = outSoltabs[indx].split('/')[1]
if not outSolset in H.getSolsets().keys():
solsetTEC = H.makeSolset(outSolset)
dirs_pos = source_positions
sourceTable = solsetTEC._f_get_child('source')
sourceTable.append(zip(*(dirs_out, dirs_pos)))
ants_pos = station_positions
antennaTable = solsetTEC._f_get_child('antenna')
antennaTable.append(zip(*(ants_out, ants_pos)))
# Store tecscreen values. The residual values are stored in the weights
# table. Flagged values of the screen have weights set to 0.0.
vals = tec_screen.transpose([1, 0, 2])
weights = residual.transpose([1, 0, 2])
tec_screen_st = H.makeSoltab(outSolset, 'tecscreen', outSoltab,
axesNames=['dir', 'time', 'ant'], axesVals=[dirs_out, times_out,
ants_out], vals=vals, weights=weights)
# Store beta, r_0, height, and order as attributes of the tecscreen
# soltab
tec_screen_st._v_attrs['beta'] = beta
tec_screen_st._v_attrs['r_0'] = r_0
tec_screen_st._v_attrs['height'] = height
tec_screen_st._v_attrs['order'] = order
# Make output piercepoint table
tec_screen_solset = tec_screen_st._v_parent._v_name
H.H.create_carray('/'+tec_screen_solset+'/'+tec_screen_st._v_name,
'piercepoint', obj=pp)
# Add histories
sw = solWriter(tec_screen_st)
sw.addHistory('CREATE (by TECSCREEN operation)')
indx += 1
return 0
def run( step, parset, H ):
"""
Separate phase solutions into Clock and TEC.
Phase solutions are assumed to be stored in solsets of the H5parm file, one
solset per field.
The Clock and TEC values are stored in the specified output soltab with type 'clock' and 'tec'.
"""
import numpy as np
from h5parm import solFetcher, solWriter
ants = getParAxis( step, parset, H, 'ant' )
logging.info('Ant: '+str(ants))
pols = getParAxis( step, parset, H, 'pol' )
logging.info('Pol: '+str(pols))
dirs = getParAxis( step, parset, H, 'dir' )
logging.info('Dir: '+str(dirs))
# get involved solsets using local step values or global values or all
solsets = getParSolsets( step, parset, H )
logging.info('Solset: '+str(solsets))
soltabs = getParSoltabs( step, parset, H )
logging.info('Soltab: '+str(soltabs))
solTypes = ['phase']
# do something on every soltab (use the openSoltab LoSoTo function)
#for soltab in openSoltabs( H, soltabs ):
for soltabname in soltabs:
solsetname=soltabname.split('/')[0]
soltab=H.getSoltab(solset=solsetname, soltab=soltabname.split('/')[1])
logging.info("--> Working on soltab: "+soltab._v_name)
# use the solFetcher from the H5parm lib
t = solFetcher(soltab)
tw = solWriter(soltab)
axisNames = t.getAxesNames()
logging.info("Axis names are: "+str(axisNames))
solType = t.getType()
if solType != 'phase':
logging.info("Soltab type of "+soltab._v_name+" is: "+solType," should be phase")
continue
# this will make a selection for the getValues() and getValuesIter()
t.setSelection(ant=ants, pol=pols, dir=dirs)
logging.info("Selection is: "+str(t.selection))
names=t.getAxesNames()
logging.info("axis names"+str(t.getAxesNames()))
# Collect station properties
station_dict = H.getAnt(solsetname)
stations=t.getAxisValues('ant')
station_positions = np.zeros((len(stations), 3), dtype=np.float)
for i, station_name in enumerate(stations):
station_positions[i, 0] = station_dict[station_name][0]
station_positions[i, 1] = station_dict[station_name][1]
station_positions[i, 2] = station_dict[station_name][2]
returnAxes=['ant','freq','pol','time']
for vals, coord in t.getValuesIter(returnAxes=returnAxes):
freqs=coord['freq']
ph=vals[:]
stations=coord['ant']
axes=[i for i in names if i in returnAxes]
clock,tec,offset,newstations=doFit(ph,freqs,stations,station_positions,axes)
times=coord['time']
tf_st = H.makeSoltab(solsetname, 'tec', 'tec',
axesNames=['time', 'ant','pol'], axesVals=[times, newstations,np.arange(2)],
vals=tec,
weights=np.ones_like(tec))
sw = solWriter(tf_st)
sw.addHistory('CREATE (by CLOCKTECFIT operation)')
tf_st = H.makeSoltab(solsetname, 'clock', 'clock',
axesNames=['time', 'ant','pol'], axesVals=[times, newstations,np.arange(2)],
vals=clock,
weights=np.ones_like(clock))
sw = solWriter(tf_st)
sw.addHistory('CREATE (by CLOCKTECFIT operation)')
tf_st = H.makeSoltab(solsetname, 'offset', 'phase_offset',
axesNames=['ant','pol'], axesVals=[newstations,np.arange(2)],
vals=offset,
weights=np.ones_like(offset))
sw = solWriter(tf_st)
sw.addHistory('CREATE (by CLOCKTECFIT operation)')
# Add history
return 0 # if everything went fine, otherwise 1
def collect_solutions(H, dirs=None, freq_tol=1e6, solsets=None):
"""
Collects and returns phase solutions, etc. needed for fitting
Keyword arguments:
H -- H5parm object
dirs -- list of directions to use (None => all)
freq_tol -- tolerance for grouping of phase solutions into bands (Hz)
solsets -- list of solution sets in H to search (None => all)
"""
import numpy as np
from pylab import find
import progressbar
import re
logging.info("Scanning for solutions needed for TEC fitting...")
# Determine axis lengths
sources = []
freqs = []
stations = []
if solsets is None:
solsets = H.getSolsets().keys()
N_times_max = 0
first_solset = None
for solset in solsets[:]:
logging.info(' --Scanning solution set {0}...'.format(solset))
has_phase_st = False
soltabs = H.getSoltabs(solset)
for soltab in soltabs:
# Only use soltabs with phase solutions
if 'phase' in soltabs[soltab]._v_title:
logging.info(' --Scanning solution table {0}...'.format(soltab))
has_phase_st = True
solset_sources = H.getSou(solset)
if len(solset_sources) == 1 and 'pointing' in solset_sources:
logging.info(' Found direction-independent solutions')
dir_indep = True
soln_type_dirindep = soltabs[soltab]._v_title
if first_solset is None:
first_solset = solset
else:
logging.info(' Found direction-dependent solutions')
dir_indep = False
if first_solset is None:
first_solset = solset
if not dir_indep:
soln_type_dirdep = soltabs[soltab]._v_title
logging.info(' Found sources: {0}'.format(
H.getSou(solset).keys()))
sources += H.getSou(solset).keys()
stations += H.getAnt(solset).keys()
t = solFetcher(soltabs[soltab])
solns, axis_vals = t.getValues()
times = axis_vals['time']
N_times = len(times)
freqs.append(axis_vals['freq'][0])
if N_times > N_times_max:
N_times_max = N_times
times_max = times
if not has_phase_st:
logging.info(' --No phase solutions found')
solsets.remove(solset)
# Consolidate frequencies into bands (defined by freq_tol parameter)
has_dup = True
while has_dup:
has_dup = False
for freq in freqs[:]:
nfreq = len( find( (np.array(freqs) > (freq-freq_tol)) &
(np.array(freqs) < (freq+freq_tol)) ) )
if nfreq > 1:
has_dup = True
freqs.remove(freq)
break
freqs = np.array(sorted(freqs))
N_freqs = len(freqs)
stations_set = set(stations)
sources_set = set(sources)
if 'pointing' in sources_set:
sources_set.remove('pointing')
if dirs is not None:
sources_filt = []
for dir in dirs:
if dir in sources_set:
sources_filt.append(dir)
sources_set = set(sources_filt)
source_names = np.array(sorted(list(sources_set)))
N_sources = len(set(source_names))
N_stations = len(set(stations))
N_times = N_times_max
times = times_max
logging.info('Scanning complete')
logging.info(' Number of sources: {0}'.format(N_sources))
logging.info(' Number of stations: {0}'.format(N_stations))
logging.info(' Number of times: {0}'.format(N_times))
logging.info(' Number of bands: {0}'.format(N_freqs))
if N_sources == 0 or N_stations == 0 or N_times == 0 or N_freqs == 0:
logging.error('No solutions found.')
return (None, None, None, None, None, None, None, None, None, None,
None, None)
# Initialize the arrays
freqwidths = np.zeros(N_freqs)
timewidths = np.zeros(N_times)
m = np.zeros((N_sources, N_freqs))
phases0 = np.zeros((N_sources, N_stations, N_freqs, N_times))
phases1 = np.zeros((N_sources, N_stations, N_freqs, N_times))
flags = np.ones((N_sources, N_stations, N_freqs, N_times))
source_positions = np.zeros((N_sources, 2))
solset_array_dir_dep = np.zeros((N_sources, N_freqs), dtype='|S100')
solset_array_dir_indep = np.zeros(N_freqs, dtype='|S100')
# Populate the arrays
for solset in solsets:
source_dict = H.getSou(solset)
sources = source_dict.keys()
soltabs = H.getSoltabs(solset)
for soltab in soltabs:
t = solFetcher(soltabs[soltab])
solns, axes = t.getValues()
freq = axes['freq'][0]
freq_indx = find( (np.array(freqs) > (freq-freq_tol)) &
(np.array(freqs) < (freq+freq_tol)) )
for source in sources:
if source in source_names:
source_indx = find(source_names == source)
m[source_indx, freq_indx] = 1
solset_array_dir_dep[source_indx, freq_indx] = solset
elif source=='pointing' and len(sources) == 1:
solset_array_dir_indep[freq_indx] = solset
# Collect station properties and pointing direction
station_dict = H.getAnt(first_solset)
station_names = np.array(list(stations_set))
station_positions = np.zeros((len(station_names), 3), dtype=np.float)
for i, station_name in enumerate(station_names):
station_positions[i, 0] = station_dict[station_name][0]
station_positions[i, 1] = station_dict[station_name][1]
station_positions[i, 2] = station_dict[station_name][2]
source_dict = H.getSou(first_solset)
ra = source_dict['pointing'][0]
dec = source_dict['pointing'][1]
pointing = np.array([np.cos(ra) * np.cos(dec), np.sin(ra) * np.cos(dec),
np.sin(dec)])
# Collect source positions and phase solutions for each band
logging.info('Collecting phase solutions...')
pbar = progressbar.ProgressBar(maxval=len(source_names)).start()
for i, source1 in enumerate(source_names):
for k in range(N_freqs):
if m[i, k]:
solset_name = str(solset_array_dir_dep[i, k])
soltab = H.getSoltab(solset=solset_name, soltab=soln_type_dirdep+'000')
solset_dir_indep_name = str(solset_array_dir_indep[k])
source_dict = H.getSou(solset_name)
source_positions[i, ] = source_dict[source1]
if solset_dir_indep_name != '':
soltab_dir_indep = H.getSoltab(solset=solset_dir_indep_name,
soltab=soln_type_dirindep+'000')
sf_dir_indep = solFetcher(soltab_dir_indep)
else:
sf_dir_indep = None
sf_dir_dep = solFetcher(soltab)
for l, station in enumerate(station_names):
if soln_type_dirdep == 'scalarphase':
sf_dir_dep.setSelection(ant=station, dir=source1)
else:
sf_dir_dep.setSelection(ant=station, pol='XX', dir=source1)
values_dir_dep = sf_dir_dep.getValues()
v1_phase = np.array(values_dir_dep[0]).squeeze()
times_dir_dep = values_dir_dep[1]['time']
ind = np.where(~np.isnan(v1_phase))
v1_phase = v1_phase[ind]
times_dir_dep = times_dir_dep[ind]
if sf_dir_indep is not None:
if soln_type_dirindep == 'scalarphase':
sf_dir_indep.setSelection(ant=station)
else:
sf_dir_indep.setSelection(ant=station, pol='XX')
values_dir_indep = sf_dir_indep.getValues()
v1_dir_indep = np.array(values_dir_indep[0]).squeeze()
times_dir_indep = values_dir_indep[1]['time']
ind = np.where(~np.isnan(v1_dir_indep))
v1_dir_indep = v1_dir_indep[ind]
times_dir_indep = times_dir_indep[ind]
v1_dir_indep_interp = interpolate_phase(v1_dir_indep,
times_dir_indep, times_dir_dep)
v1_phase += v1_dir_indep_interp
if len(times_dir_dep) != N_times_max:
v1_phase = interpolate_phase(v1_phase, times_dir_dep,
times_max)
phases0[i, l, k, :] = v1_phase
if soln_type_dirdep == 'scalarphase':
phases1[i, l, k, :] = v1_phase
else:
sf_dir_dep.setSelection(ant=station, pol='YY', dir=source1)
values_dir_dep = sf_dir_dep.getValues()
v1_phase = np.array(values_dir_dep[0]).squeeze()
times_dir_dep = values_dir_dep[1]['time']
ind = np.where(~np.isnan(v1_phase))
v1_phase = v1_phase[ind]
times_dir_dep = times_dir_dep[ind]
if sf_dir_indep is not None:
if soln_type_dirindep == 'scalarphase':
sf_dir_indep.setSelection(ant=station)
else:
sf_dir_indep.setSelection(ant=station, pol='YY')
values_dir_indep = sf_dir_indep.getValues()
v1_dir_indep = np.array(values_dir_indep[0]).squeeze()
times_dir_indep = values_dir_indep[1]['time']
ind = np.where(~np.isnan(v1_dir_indep))
v1_dir_indep = v1_dir_indep[ind]
times_dir_indep = times_dir_indep[ind]
v1_dir_indep_interp = interpolate_phase(v1_dir_indep,
times_dir_indep, times_dir_dep)
v1_phase += v1_dir_indep_interp
if len(times_dir_dep) != N_times_max:
v1_phase = interpolate_phase(v1_phase, times_dir_dep,
times_max)
phases1[i, l, k, :] = v1_phase
if len(times_dir_dep) != N_times_max:
# Set all weights to unity if interpolation was required
flags[i, l, k, :] = 1.0
else:
flags[i, l, k, :] = sf_dir_dep.getValues(weight=True,
retAxesVals=False)
if np.all(phases0[i, l, k, :] == 0.0):
# Check for flagged stations
flags[i, l, k, :] = 0.0
pbar.update(i)
pbar.finish()
logging.info('Collection complete')
for i, source in enumerate(source_names):
nbands = len(find(m[i, :]))
logging.info(' Source {0} has solutions in {1} bands'.format(source, nbands))
# Invert the weights to give flags (0 => unflagged, 1 => flagged)
zeroflags = np.where(flags == 0.0)
oneflags = flags.nonzero()
flags[zeroflags] = 1.0
flags[oneflags] = 0.0
return (phases0, phases1, flags, m, station_names, station_positions,
source_names, source_positions, freqs, times, pointing, soln_type_dirdep)
def run(step, parset, H):
import scipy.ndimage.filters
from h5parm import solFetcher, solWriter
soltabs = getParSoltabs(step, parset, H)
ants = getParAxis(step, parset, H, 'ant')
pols = getParAxis(step, parset, H, 'pol')
dirs = getParAxis(step, parset, H, 'dir')
axesToSmooth = parset.getStringVector(
'.'.join(["LoSoTo.Steps", step, "Axes"]), [])
FWHM = parset.getIntVector('.'.join(["LoSoTo.Steps", step, "FWHM"]), [])
if len(axesToSmooth) != len(FWHM):
logging.error("Axes and FWHM lenghts must be equal.")
return 1
for soltab in openSoltabs(H, soltabs):
sf = solFetcher(soltab)
sw = solWriter(soltab)
logging.info("Smoothing soltab: " + soltab._v_name)
sf.setSelection(ant=ants, pol=pols, dir=dirs)
for i, axis in enumerate(axesToSmooth[:]):
if axis not in sf.getAxesNames():
del axesToSmooth[i]
del FWHM[i]
logging.warning('Axis \"' + axis + '\" not found. Ignoring.')
for vals, coord in sf.getValuesIter(returnAxes=axesToSmooth):
valsnew = scipy.ndimage.filters.median_filter(vals, FWHM)
# TODO: implement flag control
# find the local mean (without any nans)
# valmean = np.ma.mean(np.ma.masked_array(vals, np.isnan(vals)), axis=0)
# print 'mean value: ' + str(valmean)
# replace any nans with median
# valbool = np.isnan(vals)
# nanindex = np.where(valbool)
# if valbool.any():
# logging.warning('Found NaNs in solutions. Replacing with local mean.')
# check if valmean is iterable
# if valmean.shape is tuple():
# np.putmask(vals, valbool, valmean)
# else:
# for x,mval in enumerate(valmean):
# np.putmask(vals[:,x], valbool[:,x], mval)
# writing back the solutions (selection on all the coord axis)
# this can be properly broacasted
coord = removeKeys(coord, axesToSmooth)
sw.setSelection(**coord)
sw.setValues(valsnew)
sw.addHistory('SMOOTH (over %s with box size = %s)' %
(axesToSmooth, FWHM))
return 0
def run( step, parset, H ):
"""
Generic unspecified step for easy expansion.
"""
import numpy as np
from h5parm import solFetcher, solWriter
# all the following are LoSoTo function to extract information from the parset
# get involved solsets using local step values or global values or all
solsets = getParSolsets( step, parset, H )
logging.info('Solset: '+str(solsets))
# get involved soltabs using local step values or global values or all
soltabs = getParSoltabs( step, parset, H )
logging.info('Soltab: '+str(soltabs))
# get list of Antennas using local step values or global values or all
ants = getParAxis( step, parset, H, 'ant' )
logging.info('Ant: '+str(ants))
# get list of Polarizations using local step values or global values or all
pols = getParAxis( step, parset, H, 'pol' )
logging.info('Pol: '+str(pols))
# get list of SolTypes using local step values or global values or all
solTypes = getParSolTypes( step, parset, H )
logging.info('SolType: '+str(solTypes))
# get list of Directions using local step values or global values or all
dirs = getParAxis( step, parset, H, 'dir' )
logging.info('Dir: '+str(dirs))
# do something on every soltab (use the openSoltab LoSoTo function)
for soltab in openSoltabs( H, soltabs ):
logging.info("--> Working on soltab: "+soltab._v_name)
# use the solFetcher from the H5parm lib
t = solFetcher(soltab)
tw = solWriter(soltab)
axisNames = t.getAxesNames()
logging.info("Axis names are: "+str(axisNames))
solType = t.getType()
logging.info("Soltab type is: "+solType)
# this will make a selection for the getValues() and getValuesIter()
t.setSelection(ant=ants, pol=pols, dir=dirs)
logging.info("Selection is: "+str(t.selection))
# find all axis values
logging.info("Antennas are: "+str(t.getAxisValues('ant')))
# but one can also use
logging.info("Antennas (other method) are: "+str(t.ant))
logging.info("Frequencies are: "+str(t.freq))
logging.info("Directions are: "+str(t.dir))
logging.info("Polarizations are: "+str(t.pol))
# try to access a non-existent axis
t.getAxisValues('nonexistantaxis')
# now get all values given this selection
logging.info("Get data using t.val")
val = t.val
logging.info("$ val is "+str(val[0,0,0,0,100]))
flag = t.weight
time = t.time
thisTime = t.time[100]
# another way to get the data is using the getValues()
logging.info("Get data using getValues()")
grid, axes = t.getValues()
# axis names
logging.info("Axes: "+str(t.getAxesNames()))
# axis shape
print axes
print [t.getAxisLen(axis) for axis in axes] # not ordered, is a dict!
def run( step, parset, H ):
"""
Fits a screen to TEC values derived by the TECFIT operation.
The TEC values are read from the specified tec-type soltab.
The results of the fit are stored in the specified tecfitwhite- and
piercepoint-type soltabs.
TEC screens can be plotted with the PLOT operation by setting PlotType =
TECScreen.
Note that the output screens are not normalized (any normalization was lost
due to the use of source-to-source phase gradients in the TECFIT operation).
Therefore, a direction-independent calibration must be done after exporting
the screens to a parmdb file, with the following settings in the BBS solve
step:
Model.Ionosphere.Enable = T
Model.Ionosphere.Type = EXPION
"""
import numpy as np
import re
from h5parm import solFetcher, solWriter
soltabs = getParSoltabs( step, parset, H )
outSoltabsTEC = parset.getStringVector('.'.join(["LoSoTo.Steps", step, "OutSoltabTEC"]), [] )
outSoltabsPP = parset.getStringVector('.'.join(["LoSoTo.Steps", step, "OutSoltabPP"]), [] )
height = np.float(parset.getString('.'.join(["LoSoTo.Steps", step, "Height"]), '200' ))
order = int(parset.getString('.'.join(["LoSoTo.Steps", step, "Order"]), '15' ))
logging.info('Using height = {0} m and order = {1}'.format(height, order))
# Load TEC values from TECFIT operation
indx = 0
for soltab in openSoltabs(H, soltabs):
if 'tec' not in soltab._v_title:
logging.warning('No TECFIT solution tables found for solution table '
'{0}'.format(soltabs[indx]))
continue
indx += 1
solset = soltabs[indx].split('/')[0]
logging.info('Using input solution table: {0}'.format(soltabs[indx]))
logging.info('Using output solution tables: {0} and {1}'.format(
outSoltabsTEC[indx], outSoltabsPP[indx]))
station_dict = H.getAnt(solset)
station_names = station_dict.keys()
station_positions = station_dict.values()
source_dict = H.getSou(solset)
source_names = source_dict.keys()
source_positions = source_dict.values()
t = solFetcher(soltab)
r, axis_vals = t.getValues()
times = axis_vals['time']
N_sources = len(source_names)
N_times = len(times)
N_stations = len(station_names)
N_piercepoints = N_sources * N_stations
rr = reshape(r.transpose([0,2,1]), [N_piercepoints, N_times])
# Find pierce points and airmass values for given screen height
pp, airmass = calculate_piercepoints(np.array(station_positions),
np.array(source_positions), np.array(times), height)
# Fit a TEC screen
r_0 = 10e3
beta = 5.0/3.0
tec_fit, tec_fit_white = fit_screen_to_tec(station_names, source_names,
pp, airmass, rr, times, height, order, r_0, beta)
# Write the results to the output solset
dirs_out = []
dirs_pos = []
for s in range(N_sources):
dirs_out.append(source_names[s])
times_out = times
ants_out = []
ants_pos = []
for s in range(N_stations):
ants_out.append(station_names[s])
# Make output tecfitwhite table
outSolsetTEC = outSoltabsTEC[indx].split('/')[0]
outSoltabTEC = outSoltabsTEC[indx].split('/')[1]
if not outSolsetTEC in H.getSolsets().keys():
solsetTEC = H.makeSolset(outSolsetTEC)
dirs_out = []
dirs_pos = []
for s in range(N_sources):
dirs_out.append(source_names[s])
dirs_pos.append(source_positions[s])
sourceTable = solsetTEC._f_get_child('source')
sourceTable.append(zip(*(dirs_out, dirs_pos)))
ants_out = []
ants_pos = []
for s in range(N_stations):
ants_out.append(station_names[s])
ants_pos.append(station_positions[s])
antennaTable = solsetTEC._f_get_child('antenna')
antennaTable.append(zip(*(ants_out, ants_pos)))
# Store tecfitwhite values. The tec_fit values are stored in the weights
# table to simplify things
vals = tec_fit_white.transpose([1, 0, 2])
weights = tec_fit.transpose([1, 0, 2])
tec_fit_st = H.makeSoltab(outSolsetTEC, 'tecfitwhite', outSoltabTEC,
axesNames=['dir', 'time', 'ant'], axesVals=[dirs_out, times_out,
ants_out], vals=vals, weights=weights)
# Store beta, r_0, height, and order as attributes of the tecfitwhite
# soltab
tec_fit_st._v_attrs['beta'] = beta
tec_fit_st._v_attrs['r_0'] = r_0
tec_fit_st._v_attrs['height'] = height
tec_fit_st._v_attrs['order'] = order
# Make output piercepoint table
outSolsetPP = outSoltabsPP[indx].split('/')[0]
outSoltabPP = outSoltabsPP[indx].split('/')[1]
if not outSolsetPP in H.getSolsets().keys():
solsetPP = H.makeSolset(outSolsetPP)
sourceTable = solsetPP._f_get_child('source')
sourceTable.append(zip(*(dirs_out, dirs_pos)))
antennaTable = solsetPP._f_get_child('antenna')
antennaTable.append(zip(*(ants_out, ants_pos)))
# Store piercepoint positions
pp_indx = range(N_piercepoints)
pp_pos_indx = range(3) # 0 -> x, 1 -> y, 2 -> z
pp_st = H.makeSoltab(outSolsetPP, 'piercepoint', outSoltabPP,
axesNames=['time', 'piercepoint', 'coord'], axesVals=[times_out, pp_indx,
pp_pos_indx], vals=pp, weights=np.ones_like(pp))
# Add histories
sw = solWriter(tec_fit_st)
sw.addHistory('CREATE (by TECSCREEN operation)')
sw = solWriter(pp_st)
sw.addHistory('CREATE (by TECSCREEN operation)')
indx += 1
return 0
def run(step, parset, H):
"""
Interpolate the solutions from one table into a destination table
"""
import itertools
import scipy.interpolate
import numpy as np
from h5parm import solFetcher, solWriter
solsets = getParSolsets(step, parset, H)
soltabs = getParSoltabs(step, parset, H)
solTypes = getParSolTypes(step, parset, H)
ants = getParAxis(step, parset, H, 'ant')
pols = getParAxis(step, parset, H, 'pol')
dirs = getParAxis(step, parset, H, 'dir')
calSoltab = parset.getString('.'.join(["LoSoTo.Steps", step, "CalSoltab"]),
'')
calDir = parset.getString('.'.join(["LoSoTo.Steps", step, "CalDir"]), '')
interpAxes = parset.getStringVector(
'.'.join(["LoSoTo.Steps", step, "InterpAxes"]), ['time', 'freq'])
interpMethod = parset.getString(
'.'.join(["LoSoTo.Steps", step, "InterpMethod"]), 'linear')
medAxis = parset.getString('.'.join(["LoSoTo.Steps", step, "MedAxis"]), '')
rescale = parset.getBool('.'.join(["LoSoTo.Steps", step, "Rescale"]),
False)
if interpMethod not in ["nearest", "linear", "cubic"]:
logging.error('Interpolation method must be nearest, linear or cubic.')
return 1
if rescale and medAxis == '':
logging.error('A medAxis is needed for rescaling.')
return 1
# open calibration table
css, cst = calSoltab.split('/')
cr = solFetcher(H.getSoltab(css, cst))
cAxesNames = cr.getAxesNames()
for soltab in openSoltabs(H, soltabs):
logging.info("--> Working on soltab: " + soltab._v_name)
tr = solFetcher(soltab)
tw = solWriter(soltab)
axesNames = tr.getAxesNames()
for i, interpAxis in enumerate(interpAxes[:]):
if interpAxis not in axesNames or interpAxis not in cAxesNames:
logging.error('Axis ' + interpAxis + ' not found. Ignoring.')
del interpAxes[i]
if medAxis not in axesNames or medAxis not in cAxesNames:
logging.error('Axis ' + medAxis + ' not found. Cannot proceed.')
return 1
tr.setSelection(ant=ants, pol=pols, dir=dirs)
for vals, coord in tr.getValuesIter(returnAxes=interpAxes):
# construct grid
coordSel = removeKeys(coord, interpAxes)
logging.debug("Working on coords:" + str(coordSel))
# change dir if sepcified
if calDir != '':
coordSel['dir'] = calDir
cr.setSelection(**coordSel)
calValues, calCoord = cr.getValues()
# fill medAxis with the median value
axis = cAxesNames.index(medAxis)
calValues = np.repeat(
np.expand_dims(np.median(calValues, axis), axis),
calValues.shape[axis], axis)
# create a list of values whose coords are calPoints
calValues = np.ndarray.flatten(calValues)
# create calibrator/target coordinates arrays
calPoints = []
targetPoints = []
for interpAxis in interpAxes:
calPoints.append(calCoord[interpAxis])
targetPoints.append(coord[interpAxis])
calPoints = np.array([x for x in itertools.product(*calPoints)])
targetPoints = np.array(
[x for x in itertools.product(*targetPoints)])
# interpolation
valsNew = scipy.interpolate.griddata(calPoints, calValues,
targetPoints, interpMethod)
# fill values outside boudaries with "nearest" solutions
if interpMethod != 'nearest':
valsNewNearest = scipy.interpolate.griddata(
calPoints, calValues, targetPoints, 'nearest')
valsNew[np.where(valsNew == np.nan)] = valsNewNearest[np.where(
valsNew == np.nan)]
valsNew = valsNew.reshape(vals.shape)
if rescale:
# rescale solutions
axis = interpAxes.index(medAxis)
valsMed = np.repeat(
np.expand_dims(np.median(vals, axis), axis),
vals.shape[axis], axis)
valsNewMed = np.repeat(
np.expand_dims(np.median(valsNew, axis), axis),
valsNew.shape[axis], axis)
valsNew = vals * valsNewMed / valsMed
#print "Rescaling by: ", valsNewMed[:,0]/valsMed[:,0]
# writing back the solutions
coord = removeKeys(coord, interpAxes)
tw.setSelection(**coord)
tw.setValues(valsNew)
tw.addHistory('INTERP (from table %s)' % (calSoltab))
return 0
def collect_solutions(H, dirs=None, freq_tol=1e6):
"""
Collects and returns phase solutions, etc. needed for fitting
"""
logging.info("Scanning for solutions needed for TEC fitting...")
# Determine axis lengths
sources = []
freqs = []
stations = []
solsets = H.getSolsets().keys()
N_times_max = 0
first_solset = None
for solset in solsets[:]:
if 'ion' in solset:
continue
logging.info(' --Scanning solution set {0}...'.format(solset))
has_phase_st = False
soltabs = H.getSoltabs(solset)
for soltab in soltabs:
# Only use soltabs with phase solutions
if 'phase' in soltabs[soltab]._v_title:
logging.info(' --Scanning solution table {0}...'.format(soltab))
has_phase_st = True
solset_sources = H.getSou(solset)
# Ignore direction-independent soltabs
if len(solset_sources) == 1 and 'pointing' in solset_sources:
logging.info(' Found direction-independent solutions')
dir_indep = True
soln_type_dirindep = soltabs[soltab]._v_title
if first_solset is None:
first_solset = solset
else:
logging.info(' Found direction-dependent solutions')
dir_indep = False
if not dir_indep:
soln_type_dirdep = soltabs[soltab]._v_title
logging.info(' Found sources: {0}'.format(H.getSou(solset).keys()))
sources += H.getSou(solset).keys()
stations += H.getAnt(solset).keys()
t = solFetcher(soltabs[soltab])
solns, axis_vals = t.getValues()
times = axis_vals['time']
N_times = len(times)
freqs.append(axis_vals['freq'][0])
if N_times > N_times_max:
N_times_max = N_times
times_max = times
if not has_phase_st:
logging.info(' --No phase solutions found')
solsets.remove(solset)
# Consolidate frequencies into bands (defined by freq_tol parameter)
has_dup = True
while has_dup:
has_dup = False
for freq in freqs[:]:
nfreq = len( find( (array(freqs) > (freq-freq_tol)) &
(array(freqs) < (freq+freq_tol)) ) )
if nfreq > 1:
has_dup = True
freqs.remove(freq)
break
freqs = array(sorted(freqs))
N_freqs = len(freqs)
stations_set = set(stations)
sources_set = set(sources)
if 'pointing' in sources_set:
sources_set.remove('pointing')
if dirs is not None:
sources_filt = []
for dir in dirs:
if dir in sources_set:
sources_filt.append(dir)
sources_set = set(sources_filt)
source_names = array(sorted(list(sources_set)))
N_sources = len(set(source_names))
N_stations = len(set(stations))
N_times = N_times_max
logging.info('Scanning complete')
logging.info(' Number of sources: {0}'.format(N_sources))
logging.info(' Number of stations: {0}'.format(N_stations))
logging.info(' Number of times: {0}'.format(N_times))
logging.info(' Number of freqs: {0}'.format(N_freqs))
# Initialize the arrays
freqwidths = zeros(N_freqs)
timewidths = zeros(N_times)
m = zeros((N_sources, N_freqs))
phases0 = zeros((N_sources, N_stations, N_freqs, N_times))
phases1 = zeros((N_sources, N_stations, N_freqs, N_times))
flags = ones((N_sources, N_stations, N_freqs, N_times))
source_positions = zeros((N_sources, 2))
solset_array_dir_dep = chararray((N_sources, N_freqs), itemsize=100)
solset_array_dir_indep = chararray((N_freqs), itemsize=100)
# Populate the arrays
for solset in solsets:
if 'ion' in solset:
continue
source_dict = H.getSou(solset)
sources = source_dict.keys()
soltabs = H.getSoltabs(solset)
# TODO: restrict to soltabs with phase and amp only (e.g., exclude tec soltabs)
for soltab in soltabs:
t = solFetcher(soltabs[soltab])
solns, axes = t.getValues()
freq = axes['freq'][0]
freq_indx = find( (array(freqs) > (freq-freq_tol)) & (array(freqs) < (freq+freq_tol)) )
for source in sources:
if source in source_names:
source_indx = find(source_names == source)
m[source_indx, freq_indx] = 1
solset_array_dir_dep[source_indx, freq_indx] = solset
elif source=='pointing' and len(sources) == 1:
solset_array_dir_indep[freq_indx] = solset
# Collect station properties and pointing direction
station_dict = H.getAnt(first_solset)
station_names = np.array(list(stations_set))
station_positions = np.zeros((len(station_names), 3), dtype=np.float)
for i, station_name in enumerate(station_names):
station_positions[i, 0] = station_dict[station_name][0]
station_positions[i, 1] = station_dict[station_name][1]
station_positions[i, 2] = station_dict[station_name][2]
source_dict = H.getSou(first_solset)
ra = source_dict['pointing'][0]
dec = source_dict['pointing'][1]
pointing = array([cos(ra) * cos(dec), sin(ra) * cos(dec), sin(dec)])
# Collect source positions and phase solutions for each band
logging.info('Collecting phase solutions...')
for i, source1 in enumerate(source_names):
logging.info(' Source {0}'.format(source1))
for k in range(N_freqs):
if m[i, k]:
solset_name = solset_array_dir_dep[i, k]
soltab = H.getSoltab(solset=solset_name, soltab=soln_type_dirdep+'000')
solset_dir_indep_name = solset_array_dir_indep[k]
source_dict = H.getSou(solset_name)
source_positions[i, ] = source_dict[source1]
if solset_dir_indep_name != '':
soltab_dir_indep = H.getSoltab(solset=solset_dir_indep_name, soltab=soln_type_dirindep+'000')
sf_dir_indep = solFetcher(soltab_dir_indep)
else:
sf_dir_indep = None
sf_dir_dep = solFetcher(soltab)
for l, station in enumerate(station_names):
if soln_type_dirdep == 'scalarphase':
sf_dir_dep.setSelection(ant=station, dir=source1)
else:
sf_dir_dep.setSelection(ant=station, pol='XX', dir=source1)
values_dir_dep = sf_dir_dep.getValues()
v1_phase = array(values_dir_dep[0]).squeeze()
times_dir_dep = values_dir_dep[1]['time']
if sf_dir_indep is not None:
if soln_type_dirindep == 'scalarphase':
sf_dir_indep.setSelection(ant=station)
else:
sf_dir_indep.setSelection(ant=station, pol='XX')
values_dir_indep = sf_dir_indep.getValues()
v1_dir_indep = array(values_dir_indep[0]).squeeze()
times_dir_indep = values_dir_indep[1]['time']
v1_dir_indep_interp = interpolate_phase(v1_dir_indep, times_dir_indep, times_dir_dep)
v1_phase += v1_dir_indep_interp
if len(times_dir_dep) != N_times_max:
phases0 = interpolate_phase(phases0, times_dir_indep, times_max)
phases0[i, l, k, :] = v1_phase
if soln_type_dirdep == 'scalarphase':
phases1[i, l, k, :] = v1_phase
else:
sf_dir_dep.setSelection(ant=station, pol='YY', dir=source1)
values_dir_dep = sf_dir_dep.getValues()
v1_phase = array(values_dir_dep[0]).squeeze()
times_dir_dep = values_dir_dep[1]['time']
if sf_dir_indep is not None:
if soln_type_dirindep == 'scalarphase':
sf_dir_indep.setSelection(ant=station)
else:
sf_dir_indep.setSelection(ant=station, pol='YY')
values_dir_indep = sf_dir_indep.getValues()
v1_dir_indep = array(values_dir_indep[0]).squeeze()
times_dir_indep = values_dir_indep[1]['time']
v1_dir_indep_interp = interpolate_phase(v1_dir_indep, times_dir_indep, times_dir_dep)
v1_phase += v1_dir_indep_interp
if len(times_dir_dep) != N_times_max:
phases1 = interpolate_phase(phases1, times_dir_indep, times_max)
phases1[i, l, k, :] = v1_phase
flags[i, l, k, :] = sf_dir_dep.getValues(weight=True, retAxesVals=False)
# Invert the weights to give flags (0 => unflagged, 1 => flagged)
zeroflags = np.where(flags == 0.0)
oneflags = flags.nonzero()
flags[zeroflags] = 1.0
flags[oneflags] = 0.0
return (phases0, phases1, flags, m, station_names, station_positions,
source_names, source_positions, freqs, times, pointing, soln_type_dirdep)