def run( step, parset, H ):
"""
Copy values from a table to another (of the same kind)
If tables have different sampling, then resample the values
"""
import numpy as np
from losoto.h5parm import solFetcher, solWriter
inTable = parset.getString('.'.join(["LoSoTo.Steps", step, "InTable"]), '' ) # complete solset/soltab
outTable = parset.getString('.'.join(["LoSoTo.Steps", step, "OutTable"]), '' ) # complete solset/soltab or ''
if inTable == '':
logging.error('InTable is undefined.')
return 1
if outTable == '':
outSolsetName = inTable.split('/')[0]
outTableName = None
else:
outSolsetName = outTable.split('/')[0]
outTableName = outTable.split('/')[1]
ss, st = inTable.split('/')
sf = solFetcher(H.getSoltab(ss, st))
t = H.makeSoltab(solset = outSolsetName, soltype = sf.getType(), soltab = outTableName, axesNames=sf.getAxesNames(), \
axesVals=[sf.getAxisValues(axisName) for axisName in sf.getAxesNames()], \
vals=sf.getValues(retAxesVals = False), weights=sf.getValues(weight = True, retAxesVals = False), parmdbType=sf.t._v_attrs['parmdb_type'])
sw = solWriter(t)
sw.addHistory('DUPLICATE (from table %s)' % (inTable))
return 0
def run(step, parset, H):
"""
Copy values from a table to another (of the same kind)
If tables have different sampling, then resample the values
"""
import numpy as np
from losoto.h5parm import solFetcher, solWriter
inTable = parset.getString('.'.join(["LoSoTo.Steps", step, "InTable"]),
'') # complete solset/soltab
outTable = parset.getString('.'.join(["LoSoTo.Steps", step, "OutTable"]),
'') # complete solset/soltab or ''
if inTable == '':
logging.error('InTable is undefined.')
return 1
if outTable == '':
outSolsetName = inTable.split('/')[0]
outTableName = None
else:
outSolsetName = outTable.split('/')[0]
outTableName = outTable.split('/')[1]
ss, st = inTable.split('/')
sf = solFetcher(H.getSoltab(ss, st))
t = H.makeSoltab(solset = outSolsetName, soltype = sf.getType(), soltab = outTableName, axesNames=sf.getAxesNames(), \
axesVals=[sf.getAxisValues(axisName) for axisName in sf.getAxesNames()], \
vals=sf.getValues(retAxesVals = False), weights=sf.getValues(weight = True, retAxesVals = False), parmdbType=sf.t._v_attrs['parmdb_type'])
sw = solWriter(t)
sw.addHistory('DUPLICATE (from table %s)' % (inTable))
return 0
def run(step, parset, H):
import numpy as np
from losoto.h5parm import solFetcher, solWriter
soltabs = getParSoltabs(step, parset, H)
for soltab in openSoltabs(H, soltabs):
logging.info("Taking ABSolute value of soltab: " + soltab._v_name)
sf = solFetcher(soltab)
sw = solWriter(soltab)
# axis selection
userSel = {}
for axis in sf.getAxesNames():
userSel[axis] = getParAxis(step, parset, H, axis)
sf.setSelection(**userSel)
vals = sf.getValues(retAxesVals=False)
count = np.count_nonzero(vals < 0)
logging.info('Abs: %i points initially negative (%f %%)' %
(count, 100 * float(count) / np.count_nonzero(vals)))
# writing back the solutions
sw.setValues(np.abs(vals))
sw.addHistory('ABSolute value taken')
return 0
def run( step, parset, H ):
import numpy as np
from losoto.h5parm import solFetcher, solWriter
soltabs = getParSoltabs( step, parset, H )
for soltab in openSoltabs( H, soltabs ):
logging.info("Taking ABSolute value of soltab: "+soltab._v_name)
sf = solFetcher(soltab)
sw = solWriter(soltab)
# axis selection
userSel = {}
for axis in sf.getAxesNames():
userSel[axis] = getParAxis( step, parset, H, axis )
sf.setSelection(**userSel)
vals = sf.getValues(retAxesVals = False)
count = np.count_nonzero(vals<0)
logging.info('Abs: %i points initially negative (%f %%)' % (count,100*float(count)/np.count_nonzero(vals)))
# writing back the solutions
sw.setValues(np.abs(vals))
sw.addHistory('ABSolute value taken')
return 0
def run(step, parset, H):
import scipy.ndimage.filters
import numpy as np
from losoto.h5parm import solFetcher, solWriter
soltabs = getParSoltabs(step, parset, H)
s = parset.getIntVector('.'.join(["LoSoTo.Steps", step, "smooth"]), 10)
for soltab in openSoltabs(H, soltabs):
logging.info("Smoothing soltab: " + soltab._v_name)
sf = solFetcher(soltab)
sw = solWriter(soltab)
if sf.getType() != 'clock':
logging.error(
'Only clock-type solutions can be run in this operation.')
return 1
# axis selection
userSel = {}
for axis in sf.getAxesNames():
userSel[axis] = getParAxis(step, parset, H, axis)
sf.setSelection(**userSel)
for vals, weights, coord, selection in sf.getValuesIter(
returnAxes='time', weight=True):
x = coord['time'][weights != 0]
y = vals[weights != 0]
weights = weights[weights != 0]
spline = scipy.interpolate.UnivariateSpline(x,
y,
weights,
k=1,
s=1e-15)
plot = True
if plot:
import matplotlib as mpl
mpl.use("Agg")
import matplotlib.pyplot as plt
plt.plot(x, y, 'k.')
plt.plot(x, spline(x), 'r-')
plt.savefig('test.png')
sys.exit(1)
sw.selection = selection
sw.setValues(spline(coord['time']))
sw.addHistory('Smoothed with SMOOTHCLOCK.')
del sf
del sw
return 0
def run(step, parset, H):
from losoto.h5parm import solFetcher, solWriter
soltabs = getParSoltabs(step, parset, H)
axesToExt = parset.getStringVector(
'.'.join(["LoSoTo.Steps", step, "Axes"]), ['freq', 'time'])
size = parset.getIntVector('.'.join(["LoSoTo.Steps", step, "Size"]),
[11, 11])
percent = parset.getFloat('.'.join(["LoSoTo.Steps", step, "Percent"]), 50)
cycles = parset.getInt('.'.join(["LoSoTo.Steps", step, "Cycles"]), 3)
ncpu = parset.getInt('.'.join(["LoSoTo.Ncpu"]), 1)
if axesToExt == []:
logging.error("Please specify at least one axis to extend flag.")
return 1
# start processes for multi-thread
mpm = multiprocManager(ncpu, flag)
for soltab in openSoltabs(H, soltabs):
logging.info("Extending flag on soltab: " + soltab._v_name)
sf = solFetcher(soltab)
sw = solWriter(soltab)
# axis selection
userSel = {}
for axis in sf.getAxesNames():
userSel[axis] = getParAxis(step, parset, H, axis)
sf.setSelection(**userSel)
for axisToExt in axesToExt:
if axisToExt not in sf.getAxesNames():
logging.error('Axis \"' + axisToExt + '\" not found.')
return 1
# fill the queue (note that sf and sw cannot be put into a queue since they have file references)
for vals, weights, coord, selection in sf.getValuesIter(
returnAxes=axesToExt, weight=True):
mpm.put(
[weights, coord, axesToExt, selection, percent, size, cycles])
mpm.wait()
logging.info('Writing solutions')
for w, sel in mpm.get():
sw.selection = sel
sw.setValues(w, weight=True) # convert back to np.float16
sw.addHistory('FLAG EXTENDED (over %s)' % (str(axesToExt)))
del sf
del sw
return 0
def run( step, parset, H ):
from losoto.h5parm import solFetcher, solWriter
soltabs = getParSoltabs( step, parset, H )
axesToExt = parset.getStringVector('.'.join(["LoSoTo.Steps", step, "Axes"]), ['freq','time'] )
size = parset.getIntVector('.'.join(["LoSoTo.Steps", step, "Size"]), [11,11] )
percent = parset.getFloat('.'.join(["LoSoTo.Steps", step, "Percent"]), 50 )
cycles = parset.getInt('.'.join(["LoSoTo.Steps", step, "Cycles"]), 3 )
ncpu = parset.getInt('.'.join(["LoSoTo.Ncpu"]), 0 )
if ncpu == 0:
import multiprocessing
ncpu = multiprocessing.cpu_count()
if axesToExt == []:
logging.error("Please specify at least one axis to extend flag.")
return 1
for soltab in openSoltabs( H, soltabs ):
# start processes for multi-thread
mpm = multiprocManager(ncpu, flag)
logging.info("Extending flag on soltab: "+soltab._v_name)
sf = solFetcher(soltab)
sw = solWriter(soltab)
# axis selection
userSel = {}
for axis in sf.getAxesNames():
userSel[axis] = getParAxis( step, parset, H, axis )
sf.setSelection(**userSel)
for axisToExt in axesToExt:
if axisToExt not in sf.getAxesNames():
logging.error('Axis \"'+axisToExt+'\" not found.')
mpm.wait()
return 1
# fill the queue (note that sf and sw cannot be put into a queue since they have file references)
for vals, weights, coord, selection in sf.getValuesIter(returnAxes=axesToExt, weight=True):
mpm.put([weights, coord, axesToExt, selection, percent, size, cycles])
mpm.wait()
logging.info('Writing solutions')
for w,sel in mpm.get():
sw.selection = sel
sw.setValues(w, weight=True) # convert back to np.float16
sw.addHistory('FLAG EXTENDED (over %s)' % (str(axesToExt)))
del sf
del sw
return 0
def run(step, parset, H):
"""
Normalize the solutions to a given value
"""
import numpy as np
from losoto.h5parm import solFetcher, solWriter
soltabs = getParSoltabs(step, parset, H)
solTypes = getParSolTypes(step, parset, H)
normVal = parset.getFloat('.'.join(["LoSoTo.Steps", step, "NormVal"]), 1.)
normAxes = parset.getStringVector(
'.'.join(["LoSoTo.Steps", step, "NormAxes"]), ['time'])
for soltab in openSoltabs(H, soltabs):
logging.info("Normalizing soltab: " + soltab._v_name)
tr = solFetcher(soltab)
tw = solWriter(soltab, useCache=True) # remember to flush!
axesNames = tr.getAxesNames()
for normAxis in normAxes:
if normAxis not in axesNames:
logging.error('Normalization axis ' + normAxis + ' not found.')
return 1
# axis selection
userSel = {}
for axis in tr.getAxesNames():
userSel[axis] = getParAxis(step, parset, H, axis)
tr.setSelection(**userSel)
for vals, weights, coord, selection in tr.getValuesIter(
returnAxes=normAxes, weight=True):
# rescale solutions
if np.sum(weights) == 0: continue # skip flagged antenna
valsMean = np.average(vals, weights=weights)
valsNew = normVal * vals / valsMean
logging.debug(str(coord))
logging.debug("Rescaling by: " + str(normVal / valsMean))
# writing back the solutions
tw.selection = selection
tw.setValues(valsNew)
tw.flush()
tw.addHistory('NORM (on axis %s)' % (normAxes))
return 0
def run( step, parset, H ):
import scipy.ndimage.filters
import numpy as np
from losoto.h5parm import solFetcher, solWriter
soltabs = getParSoltabs( step, parset, H )
s = parset.getIntVector('.'.join(["LoSoTo.Steps", step, "smooth"]), 10 )
for soltab in openSoltabs( H, soltabs ):
logging.info("Smoothing soltab: "+soltab._v_name)
sf = solFetcher(soltab)
sw = solWriter(soltab)
if sf.getType() != 'clock':
logging.error('Only clock-type solutions can be run in this operation.')
return 1
# axis selection
userSel = {}
for axis in sf.getAxesNames():
userSel[axis] = getParAxis( step, parset, H, axis )
sf.setSelection(**userSel)
for vals, weights, coord, selection in sf.getValuesIter(returnAxes='time', weight=True):
x=coord['time'][weights != 0]
y=vals[weights != 0]
weights = weights[weights != 0]
spline = scipy.interpolate.UnivariateSpline(x, y, weights, k=1, s=1e-15)
plot = True
if plot:
import matplotlib as mpl
mpl.use("Agg")
import matplotlib.pyplot as plt
plt.plot(x, y, 'k.')
plt.plot(x, spline(x), 'r-')
plt.savefig('test.png')
sys.exit(1)
sw.selection = selection
sw.setValues(spline(coord['time']))
sw.addHistory('Smoothed with SMOOTHCLOCK.')
del sf
del sw
return 0
def run( step, parset, H ):
"""
Normalize the solutions to a given value
"""
import numpy as np
from losoto.h5parm import solFetcher, solWriter
soltabs = getParSoltabs( step, parset, H )
solTypes = getParSolTypes( step, parset, H )
normVal = parset.getFloat('.'.join(["LoSoTo.Steps", step, "NormVal"]), 1. )
normAxes = parset.getStringVector('.'.join(["LoSoTo.Steps", step, "NormAxes"]), ['time'] )
for soltab in openSoltabs( H, soltabs ):
logging.info("Normalizing soltab: "+soltab._v_name)
tr = solFetcher(soltab)
tw = solWriter(soltab, useCache = True) # remember to flush!
axesNames = tr.getAxesNames()
for normAxis in normAxes:
if normAxis not in axesNames:
logging.error('Normalization axis '+normAxis+' not found.')
return 1
# axis selection
userSel = {}
for axis in tr.getAxesNames():
userSel[axis] = getParAxis( step, parset, H, axis )
tr.setSelection(**userSel)
for vals, weights, coord, selection in tr.getValuesIter(returnAxes=normAxes, weight = True):
# rescale solutions
if np.sum(weights) == 0: continue # skip flagged selections
valsMean = np.average(vals, weights=weights)
vals[weights != 0] *= normVal/valsMean
logging.debug(str(coord))
logging.debug("Rescaling by: "+str(normVal/valsMean))
# writing back the solutions
tw.selection = selection
tw.setValues(vals)
tw.flush()
tw.addHistory('NORM (on axis %s)' % (normAxes))
return 0
def run( step, parset, H ):
import scipy.ndimage.filters
import numpy as np
from losoto.h5parm import solFetcher, solWriter
from scipy.optimize import minimize
import itertools
from scipy.interpolate import griddata
import scipy.cluster.vq as vq
def robust_std(data, sigma=3):
"""
Calculate standard deviation excluding outliers
ok with masked arrays
"""
return np.std(data[np.where(np.abs(data) < sigma * np.std(data))])
def mask_interp(vals, mask, method='nearest'):
"""
return interpolated values for masked elements
"""
this_vals = vals.copy()
#this_vals[mask] = np.interp(np.where(mask)[0], np.where(~mask)[0], vals[~mask])
#this_vals[mask] = griddata(np.where(~mask)[0], vals[~mask], np.where(mask)[0], method)
# griddata has nan bug with nearest, I need to use vq
code, dist = vq.vq(np.where(mask)[0], np.where(~mask)[0])
this_vals[ np.where(mask)[0] ] = this_vals[code]
return this_vals
tec_jump_val = 0.019628
maxsize = 300
clip = 10 # TECs over these amount of jumps are flagged
soltabs = getParSoltabs( step, parset, H )
for soltab in openSoltabs( H, soltabs ):
logging.info("Removing TEC jumps from soltab: "+soltab._v_name)
sf = solFetcher(soltab)
sw = solWriter(soltab) # remember to flush!
# TODO: check if it's a Tec table
# axis selection
userSel = {}
for axis in sf.getAxesNames():
userSel[axis] = getParAxis( step, parset, H, axis )
sf.setSelection(**userSel)
for vals, weights, coord, selection in sf.getValuesIter(returnAxes='time', weight=True):
# skip all flagged
if (weights == 0).all(): continue
# skip reference
if (np.diff(vals[(weights == 1)]) == 0).all(): continue
# kill large values
# weights[abs(vals/tec_jump_val)>clip] = 0
# interpolate flagged values to get resonable distances
# vals = mask_interp(vals, mask=(weights == 0))/tec_jump_val
# add edges to allow intervals to the borders
# vals = np.insert(vals, 0, vals[0])
# vals = np.insert(vals, len(vals), vals[-1])
vals = np.fmod(vals,tec_jump_val)
# def find_jumps(d_vals):
# # jump poistion finder
# d_smooth = scipy.ndimage.filters.median_filter( mask_interp(d_vals, mask=(abs(d_vals)>0.8)), 21 )
# d_vals -= d_smooth
# jumps = list(np.where(np.abs(d_vals) > 1.)[0])
# return [0]+jumps+[len(d_vals)-1] # add edges
#
# class Jump(object):
# def __init__(self, jumps_idx, med):
# self.idx_left = jumps_idx[0]
# self.idx_right = jumps_idx[1]
# self.jump_left = np.rint(d_vals[self.idx_left])
# self.jump_right = np.rint(d_vals[self.idx_right])
# self.size = self.idx_right-self.idx_left
# self.hight = np.median(vals[self.idx_left+1:self.idx_right+1]-med)
# if abs((self.hight-self.jump_left)-med) > abs((self.hight-self.jump_right)-med):
# self.closejump = self.jump_right
# else:
# self.closejump = self.jump_left
#
# i = 0
# while i<len(coord['time']):
# # get tec[i] - tec[i+1], i.e. the derivative assuming constant timesteps
# # this is in units of tec_jump_val!
# d_vals = np.diff(vals)
# # get jumps idx, idx=n means a jump beteen val n and n+1
# jumps_idx = find_jumps(d_vals)
#
# # get regions
# med = np.median(vals)
# jumps = [Jump(jump_idx, med) for jump_idx in zip( jumps_idx[:-1], jumps_idx[1:] )]
# jumps = [jump for jump in jumps if jump.closejump != 0]
# jumps = [jump for jump in jumps if jump.size != 0] # prevent bug on edges
# jumps = [jump for jump in jumps if jump.size < maxsize]
#
# jumps.sort(key=lambda x: (np.abs(x.size), x.hight), reverse=False) #smallest first
# #print [(j.hight, j.closejump) for j in jumps]
#
# plot = False
# if plot:
# import matplotlib.pyplot as plt
# fig, ((ax1, ax2, ax3)) = plt.subplots(3, 1, sharex=True)
# fig.subplots_adjust(hspace=0)
# d_smooth = scipy.ndimage.filters.median_filter( mask_interp(d_vals, mask=(abs(d_vals)>0.8)), 31 )
# ax1.plot(d_vals,'k-')
# ax2.plot(d_smooth,'k-')
# ax3.plot(vals, 'k-')
# [ax3.axvline(jump_idx+0.5, color='r', ls=':') for jump_idx in jumps_idx]
# ax1.set_ylabel('d_vals')
# ax2.set_ylabel('d_vals - smooth')
# ax3.set_ylabel('TEC/jump')
# ax3.set_xlabel('timestep')
# ax1.set_xlim(xmin=-10, xmax=len(d_smooth)+10)
# fig.savefig('plots/%stecjump_debug_%03i' % (coord['ant'], i))
# i+=1
#
# if len(jumps) == 0:
# break
#
# # move down the highest to the side closest to the median
# j = jumps[0]
# #print j.idx_left, j.idx_right, j.jump_left, j.jump_right, j.hight, j.closejump
#
# vals[j.idx_left+1:j.idx_right+1] -= j.closejump
# logging.debug("%s: Number of jumps left: %i - Removing jump: %i - Size %i" % (coord['ant'], len(jumps_idx)-2, j.closejump, j.size))
# re-create proper vals
# vals = vals[1:-1]*tec_jump_val
# set back to 0 the values for flagged data
vals[weights == 0] = 0
sw.selection = selection
sw.setValues(vals)
sw.setValues(weights, weight=True)
sw.addHistory('TECJUMP')
del sf
del sw
return 0
axesVals = [['a','b','c','d'], np.arange(10), np.arange(100)]
vals = np.arange(4*10*100).reshape(4,10,100)
logging.info("Create soltab")
H5.makeSoltab(ss, 'amplitude', 'stTest', axesNames=['axis1','axis2','axis3'], axesVals=axesVals, vals=vals, weights=vals)
logging.info("Create soltab (using same name)")
H5.makeSoltab(ss, 'amplitude', 'stTest', axesNames=['axis1','axis2','axis3'], axesVals=axesVals, vals=vals, weights=vals)
logging.info("Create soltab (using default name)")
H5.makeSoltab(ss, 'amplitude', axesNames=['axis1','axis2','axis3'], axesVals=axesVals, vals=vals, weights=vals)
logging.info('Get a soltab object')
st=H5.getSoltab(ss,'stTest')
logging.info('Get all soltabs:')
print H5.getSoltabs(ss)
print "###########################################"
logging.info('### solFetcher/solWriter - General')
Hsf = solFetcher(st)
Hsw = solWriter(st)
logging.info('Get solution Type (exp: amplitude)')
print Hsf.getType()
logging.info('Get Axes Names')
print Hsf.getAxesNames()
logging.info('Get Axis1 Len (exp: 4)')
print Hsf.getAxisLen('axis1')
logging.info('Get Axis1 Type (exp: str)')
print Hsf.getAxisType('axis1')
logging.info('Get Axis2 Type (exp: float)')
print Hsf.getAxisType('axis2')
logging.info('Get Axis1 Values (exp: a,b,c,d)')
print Hsf.getAxisValues('axis1')
logging.info('Set new axes values')
Hsw.setAxisValues('axis1',['e','f','g','h'])
def run( step, parset, H ):
import os
import numpy as np
from losoto.h5parm import solFetcher, solHandler
# avoids error if re-setting "agg" a second run of plot
if not 'matplotlib' in sys.modules:
import matplotlib as mpl
mpl.rc('font',size =8 )
mpl.rc('figure.subplot',left=0.05, bottom=0.05, right=0.95, top=0.95,wspace=0.22, hspace=0.22 )
mpl.use("Agg")
import matplotlib.pyplot as plt # after setting "Agg" to speed up
soltabs = getParSoltabs( step, parset, H )
minZ, maxZ = parset.getDoubleVector('.'.join(["LoSoTo.Steps", step, "MinMax"]), [0,0] )
prefix = parset.getString('.'.join(["LoSoTo.Steps", step, "Prefix"]), '' )
# Plot various TEC-screen properties
for st_scr in openSoltabs(H, soltabs):
# Check if soltab is a tecscreen table
full_name = st_scr._v_parent._v_name + '/' + st_scr._v_name
if st_scr._v_title != 'tecscreen':
logging.warning('Solution table {0} is not a tecscreen solution '
'table. Skipping.'.format(full_name))
continue
logging.info('Using input solution table: {0}'.format(full_name))
# Plot TEC screens as images
solset = st_scr._v_parent
sf_scr = solFetcher(st_scr)
r, axis_vals = sf_scr.getValues()
source_names = axis_vals['dir']
station_names = axis_vals['ant']
station_dict = H.getAnt(solset)
station_positions = []
for station in station_names:
station_positions.append(station_dict[station])
times = axis_vals['time']
tec_screen, axis_vals = sf_scr.getValues()
times = axis_vals['time']
residuals = sf_scr.getValues(weight=True, retAxesVals=False)
height = st_scr._v_attrs['height']
order = st_scr._v_attrs['order']
beta_val = st_scr._v_attrs['beta']
r_0 = st_scr._v_attrs['r_0']
pp = sf_scr.t.piercepoint
if (minZ == 0 and maxZ == 0):
min_tec = None
max_tec = None
else:
min_tec = minZ
max_tec = maxZ
make_tec_screen_plots(pp, tec_screen, residuals,
np.array(station_positions), np.array(source_names), times,
height, order, beta_val, r_0, prefix=prefix,
remove_gradient=True, show_source_names=False, min_tec=min_tec,
max_tec=max_tec)
return 0
def run(step, parset, H):
from losoto.h5parm import solFetcher, solWriter
soltabs = getParSoltabs(step, parset, H)
#check_parset('Axes','MaxCycles','MaxRms','Order','Replace','PreFlagZeros')
axesToFlag = parset.getStringVector(
'.'.join(["LoSoTo.Steps", step, "Axes"]), 'time')
maxCycles = parset.getInt('.'.join(["LoSoTo.Steps", step, "MaxCycles"]), 5)
maxRms = parset.getFloat('.'.join(["LoSoTo.Steps", step, "MaxRms"]), 5.)
fixRms = parset.getFloat('.'.join(["LoSoTo.Steps", step, "FixRms"]), 0)
order = parset.getIntVector('.'.join(["LoSoTo.Steps", step, "Order"]), 3)
replace = parset.getBool('.'.join(["LoSoTo.Steps", step, "Replace"]),
False)
preflagzeros = parset.getBool(
'.'.join(["LoSoTo.Steps", step, "PreFlagZeros"]), False)
mode = parset.getString('.'.join(["LoSoTo.Steps", step, "Mode"]), 'smooth')
ref = parset.getString('.'.join(["LoSoTo.Steps", step, "Reference"]), '')
ncpu = parset.getInt('.'.join(["LoSoTo.Ncpu"]), 1)
if ref == '': ref = None
if axesToFlag == []:
logging.error("Please specify axis to flag. It must be a single one.")
return 1
if len(axesToFlag) != len(order):
logging.error("AxesToFlag and order must be both 1 or 2 values.")
return 1
if len(order) == 1: order = order[0]
elif len(order) == 2: order = tuple(order)
mode = mode.lower()
if mode != 'smooth' and mode != 'poly' and mode != 'spline':
logging.error('Mode must be smooth, poly or spline')
return 1
# start processes for multi-thread
mpm = multiprocManager(ncpu, flag)
for soltab in openSoltabs(H, soltabs):
logging.info("Flagging soltab: " + soltab._v_name)
sf = solFetcher(soltab)
sw = solWriter(soltab, useCache=True) # remember to flush!
# axis selection
userSel = {}
for axis in sf.getAxesNames():
userSel[axis] = getParAxis(step, parset, H, axis)
sf.setSelection(**userSel)
for axisToFlag in axesToFlag:
if axisToFlag not in sf.getAxesNames():
logging.error('Axis \"' + axis + '\" not found.')
return 1
# reorder axesToFlag as axes in the table
axesToFlag_orig = axesToFlag
axesToFlag = [
coord for coord in sf.getAxesNames() if coord in axesToFlag
]
if type(order) is int: order = [order]
if axesToFlag_orig != axesToFlag:
order = order[::-1] # reverse order if we changed axesToFlag
solType = sf.getType()
# fill the queue (note that sf and sw cannot be put into a queue since they have file references)
for vals, weights, coord, selection in sf.getValuesIter(
returnAxes=axesToFlag, weight=True, reference=ref):
mpm.put([
vals, weights, coord, solType, order, mode, preflagzeros,
maxCycles, fixRms, maxRms, replace, axesToFlag, selection
])
#v, w, sel = flag(vals, weights, coord, solType, order, mode, preflagzeros, maxCycles, fixRms, maxRms, replace, axesToFlag, selection)
mpm.wait()
for v, w, sel in mpm.get():
sw.selection = sel
if replace:
# rewrite solutions (flagged values are overwritten)
sw.setValues(v, weight=False)
else:
sw.setValues(w, weight=True)
sw.flush()
sw.addHistory('FLAG (over %s with %s sigma cut)' %
(axesToFlag, maxRms))
del sw
del sf
del soltab
return 0
def makeTECparmdb(H, solset, TECsolTab, timewidths, freq, freqwidth):
"""Returns TEC screen parmdb parameters
H - H5parm object
solset - solution set with TEC screen parameters
TECsolTab = solution table with tecscreen values
timewidths - time widths of output parmdb
freq - frequency of output parmdb
freqwidth - frequency width of output parmdb
"""
from pylab import pinv
global ipbar, pbar
station_dict = H.getAnt(solset)
station_names = station_dict.keys()
station_positions = station_dict.values()
source_dict = H.getSou(solset)
source_names = source_dict.keys()
source_positions = source_dict.values()
tec_sf = solFetcher(TECsolTab)
tec_screen, axis_vals = tec_sf.getValues()
times = axis_vals['time']
beta = TECsolTab._v_attrs['beta']
r_0 = TECsolTab._v_attrs['r_0']
height = TECsolTab._v_attrs['height']
order = TECsolTab._v_attrs['order']
pp = tec_sf.t.piercepoint
N_sources = len(source_names)
N_times = len(times)
N_freqs = 1
N_stations = len(station_names)
N_piercepoints = N_sources * N_stations
freqs = freq
freqwidths = freqwidth
parms = {}
v = {}
v['times'] = times
v['timewidths'] = timewidths
v['freqs'] = freqs
v['freqwidths'] = freqwidths
for station_name in station_names:
for source_name in source_names:
v['values'] = np.zeros((N_times, N_freqs), dtype=np.double)
parmname = 'Piercepoint:X:%s:%s' % (station_name, source_name)
parms[parmname] = v.copy()
v['values'] = np.zeros((N_times, N_freqs), dtype=np.double)
parmname = 'Piercepoint:Y:%s:%s' % (station_name, source_name)
parms[parmname] = v.copy()
v['values'] = np.zeros((N_times, N_freqs), dtype=np.double)
parmname = 'Piercepoint:Z:%s:%s' % (station_name, source_name)
parms[parmname] = v.copy()
v['values'] = np.zeros((N_times, N_freqs), dtype=np.double)
parmname = 'TECfit_white:%s:%s' % (station_name, source_name)
parms[parmname] = v.copy()
v['values'] = np.zeros((N_times, N_freqs), dtype=np.double)
parmname = 'TECfit_white:0:%s:%s' % (station_name, source_name)
parms[parmname] = v.copy()
v['values'] = np.zeros((N_times, N_freqs), dtype=np.double)
parmname = 'TECfit_white:1:%s:%s' % (station_name, source_name)
parms[parmname] = v.copy()
for k in range(N_times):
D = np.resize(pp[k, :, :], (N_piercepoints, N_piercepoints, 3))
D = np.transpose(D, (1, 0, 2)) - D
D2 = np.sum(D**2, axis=2)
C = -(D2 / (r_0**2))**(beta / 2.0) / 2.0
tec_fit_white = np.dot(pinv(C),
tec_screen[:, k, :].reshape(N_piercepoints))
pp_idx = 0
for src, source_name in enumerate(source_names):
for sta, station_name in enumerate(station_names):
parmname = 'Piercepoint:X:%s:%s' % (station_name, source_name)
parms[parmname]['values'][k, 0] = pp[k, pp_idx, 0]
parmname = 'Piercepoint:Y:%s:%s' % (station_name, source_name)
parms[parmname]['values'][k, 0] = pp[k, pp_idx, 1]
parmname = 'Piercepoint:Z:%s:%s' % (station_name, source_name)
parms[parmname]['values'][k, 0] = pp[k, pp_idx, 2]
parmname = 'TECfit_white:%s:%s' % (station_name, source_name)
parms[parmname]['values'][k, 0] = tec_fit_white[pp_idx]
parmname = 'TECfit_white:0:%s:%s' % (station_name, source_name)
parms[parmname]['values'][k, 0] = tec_fit_white[pp_idx]
parmname = 'TECfit_white:1:%s:%s' % (station_name, source_name)
parms[parmname]['values'][k, 0] = tec_fit_white[pp_idx]
pp_idx += 1
pbar.update(ipbar)
ipbar += 1
time_start = times[0] - timewidths[0] / 2
time_end = times[-1] + timewidths[-1] / 2
v['times'] = np.array([(time_start + time_end) / 2])
v['timewidths'] = np.array([time_end - time_start])
v_r0 = v.copy()
v_r0['values'] = np.array(r_0, dtype=np.double, ndmin=2)
parms['r_0'] = v_r0
v_beta = v.copy()
v_beta['values'] = np.array(beta, dtype=np.double, ndmin=2)
parms['beta'] = v_beta
v_height = v.copy()
v_height['values'] = np.array(height, dtype=np.double, ndmin=2)
parms['height'] = v_height
return parms
def run( step, parset, H ):
"""
Generic unspecified step for easy expansion.
"""
import numpy as np
from losoto.h5parm import solFetcher, solWriter
# all the following are LoSoTo function to extract information from the parset
# get involved solsets using local step values or global values or all
solsets = getParSolsets( step, parset, H )
logging.info('Solset: '+str(solsets))
# get involved soltabs using local step values or global values or all
soltabs = getParSoltabs( step, parset, H )
logging.info('Soltab: '+str(soltabs))
# get list of SolTypes using local step values or global values or all
solTypes = getParSolTypes( step, parset, H )
logging.info('SolType: '+str(solTypes))
# do something on every soltab (use the openSoltab LoSoTo function)
for soltab in openSoltabs( H, soltabs ):
logging.info("--> Working on soltab: "+soltab._v_name)
# use the solFetcher from the H5parm lib
t = solFetcher(soltab)
tw = solWriter(soltab)
axisNames = t.getAxesNames()
logging.info("Axis names are: "+str(axisNames))
solType = t.getType()
logging.info("Soltab type is: "+solType)
# this will make a selection for the getValues() and getValuesIter()
# interpret every entry in the parset which has an axis name as a selector
userSel = {}
for axis in t.getAxesNames():
userSel[axis] = getParAxis( step, parset, H, axis )
t.setSelection(**userSel)
t.setSelection(ant=ants, pol=pols, dir=dirs)
logging.info("Selection is: "+str(t.selection))
# find axis values
logging.info("Antennas (no selection) are: "+str(t.getAxisValues('ant', ignoreSelection=True)))
logging.info("Antennas (with selection) are: "+str(t.getAxisValues('ant')))
# but one can also use (selection is active here!)
logging.info("Antennas (other method) are: "+str(t.ant))
logging.info("Frequencies are: "+str(t.freq))
logging.info("Directions are: "+str(t.dir))
logging.info("Polarizations are: "+str(t.pol))
# try to access a non-existent axis
t.getAxisValues('nonexistantaxis')
# now get all values given this selection
logging.info("Get data using t.val")
val = t.val
logging.debug('shape of val: '+str(t.val.shape))
logging.info("$ val is "+str(val[0,0,0,0,100]))
weight = t.weight
time = t.time
thisTime = t.time[100]
# another way to get the data is using the getValues()
logging.info("Get data using getValues()")
grid, axes = t.getValues()
# axis names
logging.info("Axes: "+str(t.getAxesNames()))
# axis shape
print axes
print [t.getAxisLen(axis) for axis in axes] # not ordered, is a dict!
def run( step, parset, H ):
import numpy as np
from losoto.h5parm import solFetcher, solWriter
soltabs = getParSoltabs( step, parset, H )
axesToClip = parset.getStringVector('.'.join(["LoSoTo.Steps", step, "Axes"]), [] )
clipLevel = parset.getFloat('.'.join(["LoSoTo.Steps", step, "ClipLevel"]), 0. )
log = parset.getBool('.'.join(["LoSoTo.Steps", step, "Log"]), True )
if len(axesToClip) < 1:
logging.error("Please specify axes to clip.")
return 1
if clipLevel == 0.:
logging.error("Please specify factor above/below median at which to clip.")
return 1
for soltab in openSoltabs( H, soltabs ):
logging.info("Clipping soltab: "+soltab._v_name)
sf = solFetcher(soltab)
# axis selection
userSel = {}
for axis in sf.getAxesNames():
userSel[axis] = getParAxis( step, parset, H, axis )
sf.setSelection(**userSel)
# some checks
for i, axis in enumerate(axesToClip[:]):
if axis not in sf.getAxesNames():
del axesToClip[i]
logging.warning('Axis \"'+axis+'\" not found. Ignoring.')
if sf.getType() != 'amplitude':
logging.error('CLIP is for "amplitude" tables, not %s.' % sf.getType())
continue
sw = solWriter(soltab, useCache=True) # remember to flush()
before_count=0
after_count=0
total=0
for vals, weights, coord, selection in sf.getValuesIter(returnAxes=axesToClip, weight = True):
total+=len(vals)
before_count+=(len(weights)-np.count_nonzero(weights))
# first find the median and standard deviation
if (weights == 0).all():
valmedian = 0
else:
if log:
valmedian = np.median(np.log10(vals[(weights != 0)]))
rms = np.std(np.log10(vals[(weights != 0)]))
np.putmask(weights, np.abs(np.log10(vals)-valmedian) > rms * clipLevel, 0)
else:
valmedian = np.median(vals[(weights != 0)])
rms = np.std(vals[(weights != 0)])
np.putmask(weights, np.abs(vals-valmedian) > rms * clipLevel, 0)
after_count+=(len(weights)-np.count_nonzero(weights))
# writing back the solutions
sw.selection = selection
sw.setValues(weights, weight=True)
sw.addHistory('CLIP (over %s with %s sigma cut)' % (axesToClip, clipLevel))
logging.info('Clip, flagged data: %f %% -> %f %%' \
% (100.*before_count/total, 100.*after_count/total))
sw.flush()
return 0
if st._v_title == 'tecscreen':
st_tec = st
if st_tec is not None:
solTypes.append('TECScreen')
solTypes = list(set(solTypes))
logging.info('Found solution types in input parmdb and H5parm: ' +
', '.join(solTypes))
# For each solType, select appropriate solutions and construct
# the dictionary to pass to pdb.addValues()
len_sol = {}
for solType in solTypes:
if solType != 'TECScreen':
len_sol[solType] = len(pdb.getNames(solType + ':*'))
else:
tec_sf = solFetcher(st_tec)
N_times = tec_sf.getAxisLen(axis='time')
len_sol[solType] = N_times
cachedSolTabs = {}
for instrumentdbFile in instrumentdbFiles:
out_instrumentdbFile = out_globaldbFile + '/' + outroot + '_' + instrumentdbFile.split(
'/')[-1]
logging.info('Filling ' + out_instrumentdbFile + ':')
# Remove existing instrumentdb (if clobber) and create new one
if os.path.exists(out_instrumentdbFile):
if options.clobber:
shutil.rmtree(out_instrumentdbFile)
else:
logging.critical('Output instrumentdb file exists and '
opt = optparse.OptionParser(usage='%prog -p H5parm [-s solset] -g parmdb [-n 100]\n'\
+_author, version='%prog '+_version.__version__)
opt.add_option('-p', '--h5parm', help='H5parm name', type='string', default='')
opt.add_option('-g', '--parmdb', help='Parmdb name', type='string', default='')
opt.add_option('-s', '--solset', help='Solution-set name (default=sol000)', type='string', default='sol000')
opt.add_option('-n', '--numiter', help='Number of iterations (default=100)', type=int, default=100)
(options, args) = opt.parse_args()
_logging.setLevel('debug')
n = options.numiter
solset = options.solset
h5parmFile = options.h5parm
H5 = h5parm(h5parmFile, readonly=False)
H = solFetcher(H5.getSoltab(solset,'rotation000'))
logging.info("H5parm filename = "+h5parmFile)
parmdbFile = options.parmdb
P = lofar.parmdb.parmdb(parmdbFile)
P2 = lofar.parmdb.parmdb('tmp.parmdb', create=True)
logging.info("parmdb filename = "+parmdbFile)
######################################################
logging.info("### Read all frequencies for a pol/dir/station")
start = time.clock()
for i in xrange(n):
Pfreqs = P.getValuesGrid('RotationAngle:CS001LBA:3C196')['RotationAngle:CS001LBA:3C196']['freqs']
elapsed = (time.clock() - start)
logging.info("PARMDB -- "+str(elapsed)+" s.")
def run(step, parset, H):
"""
Fits a screen to TEC values derived by the TECFIT operation.
The TEC values are read from the specified tec soltab.
The results of the fit are stored in the specified tecscreen solution table.
These values are the screen TEC values per station per pierce point per
solution interval. The pierce point locations are stored in an auxiliary
array in the output solution table.
TEC screens can be plotted with the PLOT operation by setting PlotType =
TECScreen.
The H5parm_exporter.py tool can be used to export the screen to a parmdb
that BBS and the AWimager can use. Note, however, that the output screens
are not normalized properly (any normalization was lost due to the use of
source-to-source phase gradients in the TECFIT operation). Therefore, a
direction-independent calibration must be done after exporting the screens
to a parmdb file, with the following settings in the BBS solve step:
Model.Ionosphere.Enable = T
Model.Ionosphere.Type = EXPION
"""
import numpy as np
import re
from losoto.h5parm import solFetcher, solWriter
# Switch to the Agg backend to prevent problems with pylab imports when
# DISPLAY env. variable is not set
import os
if 'DISPLAY' not in os.environ:
import matplotlib
matplotlib.use("Agg")
soltabs = getParSoltabs(step, parset, H)
outSoltabs = parset.getStringVector(
'.'.join(["LoSoTo.Steps", step, "OutSoltab"]), [])
height = np.array(
parset.getDoubleVector('.'.join(["LoSoTo.Steps", step, "Height"]),
[200e3]))
order = int(
parset.getString('.'.join(["LoSoTo.Steps", step, "Order"]), '15'))
# Load TEC values from TECFIT operation
indx = 0
for soltab in openSoltabs(H, soltabs):
if 'tec' not in soltab._v_title:
logging.warning(
'No TECFIT solution tables found for solution table '
'{0}'.format(soltabs[indx]))
continue
indx += 1
solset = soltabs[indx].split('/')[0]
logging.info('Using input solution table: {0}'.format(soltabs[indx]))
logging.info('Using output solution table: {0}'.format(
outSoltabs[indx]))
# Collect station and source names and positions and times, making sure
# that they are ordered correctly.
t = solFetcher(soltab)
r, axis_vals = t.getValues()
source_names = axis_vals['dir']
source_dict = H.getSou(solset)
source_positions = []
for source in source_names:
source_positions.append(source_dict[source])
station_names = axis_vals['ant']
station_dict = H.getAnt(solset)
station_positions = []
for station in station_names:
station_positions.append(station_dict[station])
times = axis_vals['time']
# Get sizes
N_sources = len(source_names)
N_times = len(times)
N_stations = len(station_names)
N_piercepoints = N_sources * N_stations
rr = np.reshape(r.transpose([0, 2, 1]), [N_piercepoints, N_times])
heights = list(set(np.linspace(height[0], height[-1], 5)))
heights.sort()
if len(heights) > 1:
logging.info('Trying range of heights: {0} m'.format(heights))
for i, height in enumerate(heights):
# Find pierce points and airmass values for given screen height
logging.info('Using height = {0} m and order = {1}'.format(
height, order))
if height < 100e3:
logging.warning("Height is less than 100e3 m.")
pp, airmass = calculate_piercepoints(np.array(station_positions),
np.array(source_positions),
np.array(times), height)
# Fit a TEC screen
r_0 = 10e3
beta = 5.0 / 3.0
tec_screen, residual = fit_screen_to_tec(station_names,
source_names, pp, airmass,
rr, times, height, order,
r_0, beta)
total_resid = np.sum(np.abs(residual))
if i > 0:
if total_resid < best_resid:
tec_screen_best = tec_screen
pp_best = pp
height_best = height
best_resid = total_resid
else:
tec_screen_best = tec_screen
pp_best = pp
height_best = height
best_resid = total_resid
if len(heights) > 1:
logging.info(
'Total residual for fit: {0}\n'.format(total_resid))
# Use screen with lowest total residual
if len(heights) > 1:
tec_screen = tec_screen_best
pp = pp_best
height = height_best
logging.info(
'Using height (with lowest total residual) of {0} m'.format(
height))
# Write the results to the output solset
dirs_out = source_names
times_out = times
ants_out = station_names
# Make output tecscreen table
outSolset = outSoltabs[indx].split('/')[0]
outSoltab = outSoltabs[indx].split('/')[1]
if not outSolset in H.getSolsets().keys():
solsetTEC = H.makeSolset(outSolset)
dirs_pos = source_positions
sourceTable = solsetTEC._f_get_child('source')
sourceTable.append(zip(*(dirs_out, dirs_pos)))
ants_pos = station_positions
antennaTable = solsetTEC._f_get_child('antenna')
antennaTable.append(zip(*(ants_out, ants_pos)))
# Store tecscreen values. The residual values are stored in the weights
# table. Flagged values of the screen have weights set to 0.0.
vals = tec_screen.transpose([1, 0, 2])
weights = residual.transpose([1, 0, 2])
tec_screen_st = H.makeSoltab(outSolset,
'tecscreen',
outSoltab,
axesNames=['dir', 'time', 'ant'],
axesVals=[dirs_out, times_out, ants_out],
vals=vals,
weights=weights)
# Store beta, r_0, height, and order as attributes of the tecscreen
# soltab
tec_screen_st._v_attrs['beta'] = beta
tec_screen_st._v_attrs['r_0'] = r_0
tec_screen_st._v_attrs['height'] = height
tec_screen_st._v_attrs['order'] = order
# Make output piercepoint table
tec_screen_solset = tec_screen_st._v_parent._v_name
H.H.create_carray('/' + tec_screen_solset + '/' +
tec_screen_st._v_name,
'piercepoint',
obj=pp)
# Add histories
sw = solWriter(tec_screen_st)
sw.addHistory('CREATE (by TECSCREEN operation)')
indx += 1
return 0
def run( step, parset, H ):
"""
Separate phase solutions into FR, Clock and TEC.
The Clock and TEC values are stored in the specified output soltab with type 'clock', 'tec', 'FR'.
"""
from losoto.h5parm import solFetcher, solWriter
import numpy as np
import scipy.optimize
rmwavcomplex = lambda RM, wav, y: abs(np.cos(2.*RM[0]*wav*wav) - np.cos(y)) + abs(np.sin(2.*RM[0]*wav*wav) - np.sin(y))
c = 2.99792458e8
# get involved solsets using local step values or global values or all
soltabs = getParSoltabs( step, parset, H )
refAnt = parset.getString('.'.join(["LoSoTo.Steps", step, "RefAnt"]), '' )
ncpu = parset.getInt('.'.join(["LoSoTo.Ncpu"]), 1 )
for t, soltab in enumerate(openSoltabs( H, soltabs )):
logging.info("--> Working on soltab: "+soltab._v_name)
sf = solFetcher(soltab)
# times and ants needs to be complete or selection is much slower
times = sf.getAxisValues('time')
ants = sf.getAxisValues('ant')
# this will make a selection for the getValues() and getValuesIter()
userSel = {}
for axis in sf.getAxesNames():
userSel[axis] = getParAxis( step, parset, H, axis )
sf.setSelection(**userSel)
# some checks
solType = sf.getType()
if solType != 'phase':
logging.warning("Soltab type of "+soltab._v_name+" is of type "+solType+", should be phase. Ignoring.")
continue
if refAnt != '' and not refAnt in ants:
logging.error('Reference antenna '+refAnt+' not found.')
return 1
if refAnt == '': refAnt = ants[0]
solsetname = soltabs[t].split('/')[0]
st = H.makeSoltab(solsetname, 'rotationmeasure',
axesNames=['ant','time'], axesVals=[ants, times],
vals=np.zeros((len(ants),len(times))),
weights=np.ones((len(ants),len(times))))
sw = solWriter(st)
sw.addHistory('Created by FARADAY operation.')
for vals, weights, coord, selection in sf.getValuesIter(returnAxes=['freq','pol','time'], weight=True, reference = refAnt):
if len(coord['freq']) < 10:
logging.error('Faraday rotation estimation needs at least 10 frequency channels, preferably distributed over a wide range.')
return 1
fitrm = np.zeros(len(times))
fitweights = np.ones(len(times))
fitrmguess = 0 # good guess
if 'RR' in coord['pol'] and 'LL' in coord['pol']:
coord_rr = np.where(coord['pol'] == 'RR')[0][0]
coord_ll = np.where(coord['pol'] == 'LL')[0][0]
elif 'XX' in coord['pol'] and 'YY' in coord['pol']:
logging.warning('Linear polarization detected, LoSoTo assumes XX->RR and YY->LL.')
coord_rr = np.where(coord['pol'] == 'XX')[0][0]
coord_ll = np.where(coord['pol'] == 'YY')[0][0]
else:
logging.error("Cannot proceed with Faraday estimation with polarizations: "+str(coord['pol']))
return 1
if not coord['ant'] == refAnt:
logging.debug('Working on ant: '+coord['ant']+'...')
for t, time in enumerate(times):
# apply flags
idx = ((weights[0,:,t] != 0.) & (weights[1,:,t] != 0.))
freq = np.copy(coord['freq'])[idx]
phase_rr = vals[coord_rr,:,t][idx]
phase_ll = vals[coord_ll,:,t][idx]
if (len(weights[0,:,t]) - len(idx))/len(weights[0,:,t]) > 1/4.:
logging.debug('High number of filtered out data points for the timeslot '+str(t)+': '+str(len(weights[0,:,t]) - len(idx)))
if len(freq) < 10:
fitweights[t] = 0
logging.warning('No valid data found for Faraday fitting for antenna: '+coord['ant'])
continue
# RR-LL to be consistent with BBS/NDPPP
phase_diff = (phase_rr - phase_ll) # not divide by 2 otherwise jump problem, then later fix this
wav = c/freq
fitresultrm_wav, success = scipy.optimize.leastsq(rmwavcomplex, [fitrmguess], args=(wav, phase_diff))
# fractional residual
residual = np.mean(np.abs(np.mod((2.*fitresultrm_wav*wav*wav)-phase_diff,2.*np.pi) - np.pi))
# print "t:", t, "result:", fitresultrm_wav, "residual:", residual
if residual > 0.5:
fitrmguess = fitresultrm_wav[0]
weight = 1
else:
# high residual, flag
logging.warning('Bad solution for ant: '+coord['ant']+' (time: '+str(t)+', resdiaul: '+str(residual)+').')
weight = 0
# Debug plot
doplot = False
if doplot and coord['ant'] == 'RS310LBA' and t%10==0:
print "Plotting"
if not 'matplotlib' in sys.modules:
import matplotlib as mpl
mpl.rc('font',size =8 )
mpl.rc('figure.subplot',left=0.05, bottom=0.05, right=0.95, top=0.95,wspace=0.22, hspace=0.22 )
mpl.use("Agg")
import matplotlib.pyplot as plt
fig = plt.figure()
fig.subplots_adjust(wspace=0)
ax = fig.add_subplot(110)
# plot rm fit
plotrm = lambda RM, wav: np.mod( (2.*RM*wav*wav) + np.pi, 2.*np.pi) - np.pi # notice the factor of 2
ax.plot(freq, plotrm(fitresultrm_wav, c/freq[:]), "-", color='purple')
ax.plot(freq, np.mod(phase_rr + np.pi, 2.*np.pi) - np.pi, 'ob' )
ax.plot(freq, np.mod(phase_ll + np.pi, 2.*np.pi) - np.pi, 'og' )
ax.plot(freq, np.mod(phase_diff + np.pi, 2.*np.pi) - np.pi , '.', color='purple' )
residual = np.mod(plotrm(fitresultrm_wav, c/freq[:])-phase_diff+np.pi,2.*np.pi)-np.pi
ax.plot(freq, residual, '.', color='yellow')
ax.set_xlabel('freq')
ax.set_ylabel('phase')
ax.set_ylim(ymin=-np.pi, ymax=np.pi)
logging.warning('Save pic: '+str(t)+'_'+coord['ant']+'.png')
plt.savefig(str(t)+'_'+coord['ant']+'.png', bbox_inches='tight')
del fig
fitrm[t] = fitresultrm_wav[0]
fitweights[t] = weight
sw.setSelection(ant=coord['ant'], time=coord['time'])
sw.setValues( np.expand_dims(fitrm, axis=1) )
sw.setValues( np.expand_dims(fitweights, axis=1), weight=True )
del st
del sw
del sf
return 0
def run(step, parset, H):
"""
Separate phase solutions into Clock and TEC.
Phase solutions are assumed to be stored in solsets of the H5parm file, one
solset per field.
The Clock and TEC values are stored in the specified output soltab with type 'clock' and 'tec'.
"""
import numpy as np
from losoto.h5parm import solFetcher, solWriter
# get involved solsets using local step values or global values or all
soltabs = getParSoltabs(step, parset, H)
flagBadChannels = parset.getBool(
'.'.join(["LoSoTo.Steps", step, "FlagBadChannels"]), True)
flagCut = parset.getFloat('.'.join(["LoSoTo.Steps", step, "FlagCut"]), 5.)
chi2cut = parset.getFloat('.'.join(["LoSoTo.Steps", step, "Chi2cut"]),
3000.)
combinePol = parset.getBool('.'.join(["LoSoTo.Steps", step, "CombinePol"]),
False)
#fitOffset = parset.getBool('.'.join(["LoSoTo.Steps", step, "FitOffset"]), False )
removePhaseWraps = parset.getBool(
'.'.join(["LoSoTo.Steps", step, "RemovePhaseWraps"]), True)
fit3rdorder = parset.getBool(
'.'.join(["LoSoTo.Steps", step, "Fit3rdOrder"]), False)
circular = parset.getBool('.'.join(["LoSoTo.Steps", step, "Circular"]),
False)
reverse = parset.getBool('.'.join(["LoSoTo.Steps", step, "Reverse"]),
False)
# do something on every soltab (use the openSoltab LoSoTo function)
#for soltab in openSoltabs( H, soltabs ):
for soltabname in soltabs:
solsetname = soltabname.split('/')[0]
soltab = H.getSoltab(solset=solsetname,
soltab=soltabname.split('/')[1])
logging.info("--> Working on soltab: " + soltab._v_name)
t = solFetcher(soltab)
tw = solWriter(soltab)
# some checks
solType = t.getType()
if solType != 'phase':
logging.warning("Soltab type of " + soltab._v_name + " is: " +
solType + " should be phase. Ignoring.")
continue
# this will make a selection for the getValues() and getValuesIter()
userSel = {}
for axis in t.getAxesNames():
userSel[axis] = getParAxis(step, parset, H, axis)
t.setSelection(**userSel)
# Collect station properties
station_dict = H.getAnt(solsetname)
stations = t.getAxisValues('ant')
station_positions = np.zeros((len(stations), 3), dtype=np.float)
for i, station_name in enumerate(stations):
station_positions[i, 0] = station_dict[station_name][0]
station_positions[i, 1] = station_dict[station_name][1]
station_positions[i, 2] = station_dict[station_name][2]
returnAxes = ['ant', 'freq', 'pol', 'time']
for vals, flags, coord, selection in t.getValuesIter(
returnAxes=returnAxes, weight=True):
if len(coord['ant']) < 2:
logging.error(
'Clock/TEC separation needs at least 2 antennas selected.')
return 1
if len(coord['freq']) < 10:
logging.error(
'Clock/TEC separation needs at least 10 frequency channels, preferably distributed over a wide range'
)
return 1
freqs = coord['freq']
stations = coord['ant']
times = coord['time']
# get axes index
axes = [i for i in t.getAxesNames() if i in returnAxes]
# reverse time axes
if reverse:
vals = np.swapaxes(
np.swapaxes(vals, 0, axes.index('time'))[::-1], 0,
axes.index('time'))
flags = np.swapaxes(
np.swapaxes(flags, 0, axes.index('time'))[::-1], 0,
axes.index('time'))
result=doFit(vals,flags==0,freqs,stations,station_positions,axes,\
flagBadChannels=flagBadChannels,flagcut=flagCut,chi2cut=chi2cut,combine_pol=combinePol,removePhaseWraps=removePhaseWraps,fit3rdorder=fit3rdorder,circular=circular)
if fit3rdorder:
clock, tec, offset, tec3rd = result
if reverse:
clock = clock[::-1, :]
tec = tec[::-1, :]
tec3rd = tec3rd[::-1, :]
else:
clock, tec, offset = result
if reverse:
clock = clock[::-1, :]
tec = tec[::-1, :]
weights = tec > -5
tec[np.logical_not(weights)] = 0
clock[np.logical_not(weights)] = 0
weights = np.float16(weights)
if combinePol:
tf_st = H.makeSoltab(solsetname,
'tec',
axesNames=['time', 'ant'],
axesVals=[times, stations],
vals=tec[:, :, 0],
weights=weights[:, :, 0])
sw = solWriter(tf_st)
sw.addHistory('CREATE (by CLOCKTECFIT operation)')
tf_st = H.makeSoltab(solsetname,
'clock',
axesNames=['time', 'ant'],
axesVals=[times, stations],
vals=clock[:, :, 0] * 1e-9,
weights=weights[:, :, 0])
sw = solWriter(tf_st)
sw.addHistory('CREATE (by CLOCKTECFIT operation)')
tf_st = H.makeSoltab(solsetname,
'phase_offset',
axesNames=['ant'],
axesVals=[stations],
vals=offset[:, 0],
weights=np.ones_like(offset[:, 0],
dtype=np.float16))
sw = solWriter(tf_st)
sw.addHistory('CREATE (by CLOCKTECFIT operation)')
if fit3rdorder:
tf_st = H.makeSoltab(solsetname,
'tec3rd',
axesNames=['time', 'ant'],
axesVals=[times, stations],
vals=tec3rd[:, :, 0],
weights=weights[:, :, 0])
sw = solWriter(tf_st)
else:
tf_st = H.makeSoltab(solsetname,
'tec',
axesNames=['time', 'ant', 'pol'],
axesVals=[times, stations, ['XX', 'YY']],
vals=tec,
weights=weights)
sw = solWriter(tf_st)
sw.addHistory('CREATE (by CLOCKTECFIT operation)')
tf_st = H.makeSoltab(solsetname,
'clock',
axesNames=['time', 'ant', 'pol'],
axesVals=[times, stations, ['XX', 'YY']],
vals=clock * 1e-9,
weights=weights)
sw = solWriter(tf_st)
sw.addHistory('CREATE (by CLOCKTECFIT operation)')
tf_st = H.makeSoltab(solsetname,
'phase_offset',
axesNames=['ant', 'pol'],
axesVals=[stations, ['XX', 'YY']],
vals=offset,
weights=np.ones_like(offset,
dtype=np.float16))
sw = solWriter(tf_st)
sw.addHistory('CREATE (by CLOCKTECFIT operation)')
if fit3rdorder:
tf_st = H.makeSoltab(
solsetname,
'tec3rd',
axesNames=['time', 'ant', 'pol'],
axesVals=[times, stations, ['XX', 'YY']],
vals=tec3rd,
weights=weights)
sw = solWriter(tf_st)
return 0
def makeTECparmdb(H, solset, TECsolTab, PPsolTab, timewidths, freq, freqwidth):
"""Returns TEC screen parmdb parameters
H - H5parm object
solset - solution set with TEC screen parameters
TECsolTab = solution table with tecfitwhite values
PPsolTab = solution table with piercepoint values
timewidths - time widths of output parmdb
freq - frequency of output parmdb
freqwidth - frequency width of output parmdb
"""
station_dict = H.getAnt(solset)
station_names = station_dict.keys()
station_positions = station_dict.values()
source_dict = H.getSou(solset)
source_names = source_dict.keys()
source_positions = source_dict.values()
tec_sf = solFetcher(TECsolTab)
tec_fit_white, axis_vals = tec_sf.getValues()
times = axis_vals['time']
beta = TECsolTab._v_attrs['beta']
r_0 = TECsolTab._v_attrs['r_0']
height = TECsolTab._v_attrs['height']
order = TECsolTab._v_attrs['order']
pp_sf = solFetcher(PPsolTab)
pp, axis_vals = pp_sf.getValues()
N_sources = len(source_names)
N_times = len(times)
N_freqs = 1
N_stations = len(station_names)
N_piercepoints = N_sources * N_stations
freqs = freq
freqwidths = freqwidth
parms = {}
v = {}
v['times'] = times
v['timewidths'] = timewidths
v['freqs'] = freqs
v['freqwidths'] = freqwidths
for station_name in station_names:
for source_name in source_names:
v['values'] = np.zeros((N_times, N_freqs), dtype=np.double)
parmname = 'Piercepoint:X:%s:%s' % (station_name, source_name)
parms[parmname] = v.copy()
v['values'] = np.zeros((N_times, N_freqs), dtype=np.double)
parmname = 'Piercepoint:Y:%s:%s' % (station_name, source_name)
parms[parmname] = v.copy()
v['values'] = np.zeros((N_times, N_freqs), dtype=np.double)
parmname = 'Piercepoint:Z:%s:%s' % (station_name, source_name)
parms[parmname] = v.copy()
v['values'] = np.zeros((N_times, N_freqs), dtype=np.double)
parmname = 'TECfit_white:%s:%s' % (station_name, source_name)
parms[parmname] = v.copy()
v['values'] = np.zeros((N_times, N_freqs), dtype=np.double)
parmname = 'TECfit_white:0:%s:%s' % (station_name, source_name)
parms[parmname] = v.copy()
v['values'] = np.zeros((N_times, N_freqs), dtype=np.double)
parmname = 'TECfit_white:1:%s:%s' % (station_name, source_name)
parms[parmname] = v.copy()
for k in range(N_times):
pp_idx = 0
for src, source_name in enumerate(source_names):
for sta, station_name in enumerate(station_names):
parmname = 'Piercepoint:X:%s:%s' % (station_name, source_name)
parms[parmname]['values'][k, 0] = pp[k, pp_idx, 0]
parmname = 'Piercepoint:Y:%s:%s' % (station_name, source_name)
parms[parmname]['values'][k, 0] = pp[k, pp_idx, 1]
parmname = 'Piercepoint:Z:%s:%s' % (station_name, source_name)
parms[parmname]['values'][k, 0] = pp[k, pp_idx, 2]
parmname = 'TECfit_white:%s:%s' % (station_name, source_name)
parms[parmname]['values'][k, 0] = tec_fit_white[src, k, sta]
parmname = 'TECfit_white:0:%s:%s' % (station_name, source_name)
parms[parmname]['values'][k, 0] = tec_fit_white[src, k, sta]
parmname = 'TECfit_white:1:%s:%s' % (station_name, source_name)
parms[parmname]['values'][k, 0] = tec_fit_white[src, k, sta]
pp_idx += 1
time_start = times[0] - timewidths[0]/2
time_end = times[-1] + timewidths[-1]/2
v['times'] = np.array([(time_start + time_end) / 2])
v['timewidths'] = np.array([time_end - time_start])
v_r0 = v.copy()
v_r0['values'] = np.array(r_0, dtype=np.double, ndmin=2)
parms['r_0'] = v_r0
v_beta = v.copy()
v_beta['values'] = np.array(beta, dtype=np.double, ndmin=2)
parms['beta'] = v_beta
v_height = v.copy()
v_height['values'] = np.array(height, dtype=np.double, ndmin=2)
parms['height'] = v_height
return parms
def run( step, parset, H ):
"""
Separate phase solutions into Clock and TEC.
Phase solutions are assumed to be stored in solsets of the H5parm file, one
solset per field.
The Clock and TEC values are stored in the specified output soltab with type 'clock' and 'tec'.
"""
import numpy as np
from losoto.h5parm import solFetcher, solWriter
# get involved solsets using local step values or global values or all
soltabs = getParSoltabs( step, parset, H )
flagBadChannels = parset.getBool('.'.join(["LoSoTo.Steps", step, "FlagBadChannels"]), True )
flagCut = parset.getFloat('.'.join(["LoSoTo.Steps", step, "FlagCut"]), 5. )
chi2cut = parset.getFloat('.'.join(["LoSoTo.Steps", step, "Chi2cut"]), 3000. )
combinePol = parset.getBool('.'.join(["LoSoTo.Steps", step, "CombinePol"]), False )
#fitOffset = parset.getBool('.'.join(["LoSoTo.Steps", step, "FitOffset"]), False )
removePhaseWraps=parset.getBool('.'.join(["LoSoTo.Steps", step, "RemovePhaseWraps"]), True )
fit3rdorder=parset.getBool('.'.join(["LoSoTo.Steps", step, "Fit3rdOrder"]), False )
circular=parset.getBool('.'.join(["LoSoTo.Steps", step, "Circular"]), False )
reverse=parset.getBool('.'.join(["LoSoTo.Steps", step, "Reverse"]), False )
# do something on every soltab (use the openSoltab LoSoTo function)
#for soltab in openSoltabs( H, soltabs ):
for soltabname in soltabs:
solsetname=soltabname.split('/')[0]
soltab=H.getSoltab(solset=solsetname, soltab=soltabname.split('/')[1])
logging.info("--> Working on soltab: "+soltab._v_name)
t = solFetcher(soltab)
tw = solWriter(soltab)
# some checks
solType = t.getType()
if solType != 'phase':
logging.warning("Soltab type of "+soltab._v_name+" is: "+solType+" should be phase. Ignoring.")
continue
# this will make a selection for the getValues() and getValuesIter()
userSel = {}
for axis in t.getAxesNames():
userSel[axis] = getParAxis( step, parset, H, axis )
t.setSelection(**userSel)
# Collect station properties
station_dict = H.getAnt(solsetname)
stations = t.getAxisValues('ant')
station_positions = np.zeros((len(stations), 3), dtype=np.float)
for i, station_name in enumerate(stations):
station_positions[i, 0] = station_dict[station_name][0]
station_positions[i, 1] = station_dict[station_name][1]
station_positions[i, 2] = station_dict[station_name][2]
returnAxes=['ant','freq','pol','time']
for vals, flags, coord, selection in t.getValuesIter(returnAxes=returnAxes,weight=True):
if len(coord['ant']) < 2:
logging.error('Clock/TEC separation needs at least 2 antennas selected.')
return 1
if len(coord['freq']) < 10:
logging.error('Clock/TEC separation needs at least 10 frequency channels, preferably distributed over a wide range')
return 1
freqs=coord['freq']
stations=coord['ant']
times=coord['time']
# get axes index
axes=[i for i in t.getAxesNames() if i in returnAxes]
# reverse time axes
if reverse:
vals = np.swapaxes(np.swapaxes(vals, 0, axes.index('time'))[::-1], 0, axes.index('time'))
flags = np.swapaxes(np.swapaxes(flags, 0, axes.index('time'))[::-1], 0, axes.index('time'))
result=doFit(vals,flags==0,freqs,stations,station_positions,axes,\
flagBadChannels=flagBadChannels,flagcut=flagCut,chi2cut=chi2cut,combine_pol=combinePol,removePhaseWraps=removePhaseWraps,fit3rdorder=fit3rdorder,circular=circular)
if fit3rdorder:
clock,tec,offset,tec3rd=result
if reverse:
clock = clock[::-1,:]
tec = tec[::-1,:]
tec3rd = tec3rd[::-1,:]
else:
clock,tec,offset=result
if reverse:
clock = clock[::-1,:]
tec = tec[::-1,:]
weights=tec>-5
tec[np.logical_not(weights)]=0
clock[np.logical_not(weights)]=0
weights=np.float16(weights)
if combinePol:
tf_st = H.makeSoltab(solsetname, 'tec',
axesNames=['time', 'ant'], axesVals=[times, stations],
vals=tec[:,:,0],
weights=weights[:,:,0])
sw = solWriter(tf_st)
sw.addHistory('CREATE (by CLOCKTECFIT operation)')
tf_st = H.makeSoltab(solsetname, 'clock',
axesNames=['time', 'ant'], axesVals=[times, stations],
vals=clock[:,:,0]*1e-9,
weights=weights[:,:,0])
sw = solWriter(tf_st)
sw.addHistory('CREATE (by CLOCKTECFIT operation)')
tf_st = H.makeSoltab(solsetname, 'phase_offset',
axesNames=['ant'], axesVals=[stations],
vals=offset[:,0],
weights=np.ones_like(offset[:,0],dtype=np.float16))
sw = solWriter(tf_st)
sw.addHistory('CREATE (by CLOCKTECFIT operation)')
if fit3rdorder:
tf_st = H.makeSoltab(solsetname, 'tec3rd',
axesNames=['time', 'ant'], axesVals=[times, stations],
vals=tec3rd[:,:,0],
weights=weights[:,:,0])
sw = solWriter(tf_st)
else:
tf_st = H.makeSoltab(solsetname, 'tec',
axesNames=['time', 'ant','pol'], axesVals=[times, stations, ['XX','YY']],
vals=tec,
weights=weights)
sw = solWriter(tf_st)
sw.addHistory('CREATE (by CLOCKTECFIT operation)')
tf_st = H.makeSoltab(solsetname, 'clock',
axesNames=['time', 'ant','pol'], axesVals=[times, stations, ['XX','YY']],
vals=clock*1e-9,
weights=weights)
sw = solWriter(tf_st)
sw.addHistory('CREATE (by CLOCKTECFIT operation)')
tf_st = H.makeSoltab(solsetname, 'phase_offset',
axesNames=['ant','pol'], axesVals=[stations, ['XX','YY']],
vals=offset,
weights=np.ones_like(offset,dtype=np.float16))
sw = solWriter(tf_st)
sw.addHistory('CREATE (by CLOCKTECFIT operation)')
if fit3rdorder:
tf_st = H.makeSoltab(solsetname, 'tec3rd',
axesNames=['time', 'ant','pol'], axesVals=[times, stations, ['XX','YY']],
vals=tec3rd,
weights=weights)
sw = solWriter(tf_st)
return 0
def collect_solutions(H, dirs=None, freq_tol=1e6, solsets=None):
"""
Collects and returns phase solutions, etc. needed for fitting
Keyword arguments:
H -- H5parm object
dirs -- list of directions to use (None => all)
freq_tol -- tolerance for grouping of phase solutions into bands (Hz)
solsets -- list of solution sets in H to search (None => all)
"""
import numpy as np
from pylab import find
import re
try:
import progressbar
except ImportError:
import losoto.progressbar as progressbar
logging.info("Scanning for solutions needed for TEC fitting...")
# Determine axis lengths
sources = []
freqs = []
stations = []
if solsets is None:
solsets = H.getSolsets().keys()
N_times_max = 0
first_solset = None
for solset in solsets[:]:
logging.info(' --Scanning solution set {0}...'.format(solset))
has_phase_st = False
soltabs = H.getSoltabs(solset)
for soltab in soltabs:
# Only use soltabs with phase solutions
if 'phase' in soltabs[soltab]._v_title:
logging.info(
' --Scanning solution table {0}...'.format(soltab))
has_phase_st = True
solset_sources = H.getSou(solset)
if len(solset_sources) == 1 and 'pointing' in solset_sources:
logging.info(' Found direction-independent solutions')
dir_indep = True
soln_type_dirindep = soltabs[soltab]._v_title
if first_solset is None:
first_solset = solset
else:
logging.info(' Found direction-dependent solutions')
dir_indep = False
if first_solset is None:
first_solset = solset
if not dir_indep:
soln_type_dirdep = soltabs[soltab]._v_title
logging.info(' Found sources: {0}'.format(
H.getSou(solset).keys()))
sources += H.getSou(solset).keys()
stations += H.getAnt(solset).keys()
t = solFetcher(soltabs[soltab])
solns, axis_vals = t.getValues()
times = axis_vals['time']
N_times = len(times)
freqs.append(axis_vals['freq'][0])
if N_times > N_times_max:
N_times_max = N_times
times_max = times
if not has_phase_st:
logging.info(' --No phase solutions found')
solsets.remove(solset)
# Consolidate frequencies into bands (defined by freq_tol parameter)
has_dup = True
while has_dup:
has_dup = False
for freq in freqs[:]:
nfreq = len(
find((np.array(freqs) > (freq - freq_tol))
& (np.array(freqs) < (freq + freq_tol))))
if nfreq > 1:
has_dup = True
freqs.remove(freq)
break
freqs = np.array(sorted(freqs))
N_freqs = len(freqs)
stations_set = set(stations)
sources_set = set(sources)
if 'pointing' in sources_set:
sources_set.remove('pointing')
if dirs is not None:
sources_filt = []
for dir in dirs:
if dir in sources_set:
sources_filt.append(dir)
sources_set = set(sources_filt)
source_names = np.array(sorted(list(sources_set)))
N_sources = len(set(source_names))
N_stations = len(set(stations))
N_times = N_times_max
times = times_max
logging.info('Scanning complete')
logging.info(' Number of sources: {0}'.format(N_sources))
logging.info(' Number of stations: {0}'.format(N_stations))
logging.info(' Number of times: {0}'.format(N_times))
logging.info(' Number of bands: {0}'.format(N_freqs))
if N_sources == 0 or N_stations == 0 or N_times == 0 or N_freqs == 0:
logging.error('No solutions found.')
return (None, None, None, None, None, None, None, None, None, None,
None, None)
# Initialize the arrays
freqwidths = np.zeros(N_freqs)
timewidths = np.zeros(N_times)
m = np.zeros((N_sources, N_freqs))
phases0 = np.zeros((N_sources, N_stations, N_freqs, N_times))
phases1 = np.zeros((N_sources, N_stations, N_freqs, N_times))
flags = np.ones((N_sources, N_stations, N_freqs, N_times))
source_positions = np.zeros((N_sources, 2))
solset_array_dir_dep = np.zeros((N_sources, N_freqs), dtype='|S100')
solset_array_dir_indep = np.zeros(N_freqs, dtype='|S100')
# Populate the arrays
for solset in solsets:
source_dict = H.getSou(solset)
sources = source_dict.keys()
soltabs = H.getSoltabs(solset)
for soltab in soltabs:
t = solFetcher(soltabs[soltab])
solns, axes = t.getValues()
freq = axes['freq'][0]
freq_indx = find((np.array(freqs) > (freq - freq_tol))
& (np.array(freqs) < (freq + freq_tol)))
for source in sources:
if source in source_names:
source_indx = find(source_names == source)
m[source_indx, freq_indx] = 1
solset_array_dir_dep[source_indx, freq_indx] = solset
elif source == 'pointing' and len(sources) == 1:
solset_array_dir_indep[freq_indx] = solset
# Collect station properties and pointing direction
station_dict = H.getAnt(first_solset)
station_names = np.array(list(stations_set))
station_positions = np.zeros((len(station_names), 3), dtype=np.float)
for i, station_name in enumerate(station_names):
station_positions[i, 0] = station_dict[station_name][0]
station_positions[i, 1] = station_dict[station_name][1]
station_positions[i, 2] = station_dict[station_name][2]
source_dict = H.getSou(first_solset)
ra = source_dict['pointing'][0]
dec = source_dict['pointing'][1]
pointing = np.array(
[np.cos(ra) * np.cos(dec),
np.sin(ra) * np.cos(dec),
np.sin(dec)])
# Collect source positions and phase solutions for each band
logging.info('Collecting phase solutions...')
pbar = progressbar.ProgressBar(maxval=len(source_names)).start()
for i, source1 in enumerate(source_names):
for k in xrange(N_freqs):
if m[i, k]:
solset_name = str(solset_array_dir_dep[i, k])
soltab = H.getSoltab(solset=solset_name,
soltab=soln_type_dirdep + '000')
solset_dir_indep_name = str(solset_array_dir_indep[k])
source_dict = H.getSou(solset_name)
source_positions[i, ] = source_dict[source1]
if solset_dir_indep_name != '':
soltab_dir_indep = H.getSoltab(
solset=solset_dir_indep_name,
soltab=soln_type_dirindep + '000')
sf_dir_indep = solFetcher(soltab_dir_indep)
else:
sf_dir_indep = None
sf_dir_dep = solFetcher(soltab)
for l, station in enumerate(station_names):
if soln_type_dirdep == 'scalarphase':
sf_dir_dep.setSelection(ant=[station], dir=[source1])
else:
sf_dir_dep.setSelection(ant=[station],
pol=['XX'],
dir=[source1])
values_dir_dep = sf_dir_dep.getValues()
v1_phase = np.array(values_dir_dep[0]).squeeze()
times_dir_dep = values_dir_dep[1]['time']
ind = np.where(~np.isnan(v1_phase))
v1_phase = v1_phase[ind]
times_dir_dep = times_dir_dep[ind]
if sf_dir_indep is not None:
if soln_type_dirindep == 'scalarphase':
sf_dir_indep.setSelection(ant=[station])
else:
sf_dir_indep.setSelection(ant=[station],
pol=['XX'])
values_dir_indep = sf_dir_indep.getValues()
v1_dir_indep = np.array(values_dir_indep[0]).squeeze()
times_dir_indep = values_dir_indep[1]['time']
ind = np.where(~np.isnan(v1_dir_indep))
v1_dir_indep = v1_dir_indep[ind]
times_dir_indep = times_dir_indep[ind]
v1_dir_indep_interp = interpolate_phase(
v1_dir_indep, times_dir_indep, times_dir_dep)
v1_phase += v1_dir_indep_interp
if len(times_dir_dep) != N_times_max:
v1_phase = interpolate_phase(v1_phase, times_dir_dep,
times_max)
phases0[i, l, k, :] = v1_phase
if soln_type_dirdep == 'scalarphase':
phases1[i, l, k, :] = v1_phase
else:
sf_dir_dep.setSelection(ant=[station],
pol=['YY'],
dir=[source1])
values_dir_dep = sf_dir_dep.getValues()
v1_phase = np.array(values_dir_dep[0]).squeeze()
times_dir_dep = values_dir_dep[1]['time']
ind = np.where(~np.isnan(v1_phase))
v1_phase = v1_phase[ind]
times_dir_dep = times_dir_dep[ind]
if sf_dir_indep is not None:
if soln_type_dirindep == 'scalarphase':
sf_dir_indep.setSelection(ant=[station])
else:
sf_dir_indep.setSelection(ant=[station],
pol=['YY'])
values_dir_indep = sf_dir_indep.getValues()
v1_dir_indep = np.array(
values_dir_indep[0]).squeeze()
times_dir_indep = values_dir_indep[1]['time']
ind = np.where(~np.isnan(v1_dir_indep))
v1_dir_indep = v1_dir_indep[ind]
times_dir_indep = times_dir_indep[ind]
v1_dir_indep_interp = interpolate_phase(
v1_dir_indep, times_dir_indep, times_dir_dep)
v1_phase += v1_dir_indep_interp
if len(times_dir_dep) != N_times_max:
v1_phase = interpolate_phase(
v1_phase, times_dir_dep, times_max)
phases1[i, l, k, :] = v1_phase
if len(times_dir_dep) != N_times_max:
# Set all weights to unity if interpolation was required
flags[i, l, k, :] = 1.0
else:
flags[i, l,
k, :] = sf_dir_dep.getValues(weight=True,
retAxesVals=False)
if np.all(phases0[i, l, k, :] == 0.0):
# Check for flagged stations
flags[i, l, k, :] = 0.0
pbar.update(i)
pbar.finish()
logging.info('Collection complete')
for i, source in enumerate(source_names):
nbands = len(find(m[i, :]))
logging.info(' Source {0} has solutions in {1} bands'.format(
source, nbands))
# Invert the weights to give flags (0 => unflagged, 1 => flagged)
zeroflags = np.where(flags == 0.0)
oneflags = flags.nonzero()
flags[zeroflags] = 1.0
flags[oneflags] = 0.0
return (phases0, phases1, flags, m, station_names, station_positions,
source_names, source_positions, freqs, times, pointing,
soln_type_dirdep)
def run(step, parset, H):
"""
Interpolate the solutions from one table into a destination table
"""
import itertools
import scipy.interpolate
import numpy as np
from losoto.h5parm import solFetcher, solWriter
soltabs = getParSoltabs(step, parset, H)
solTypes = getParSolTypes(step, parset, H)
calSoltab = parset.getString('.'.join(["LoSoTo.Steps", step, "CalSoltab"]),
'')
calDir = parset.getString('.'.join(["LoSoTo.Steps", step, "CalDir"]), '')
interpAxes = parset.getStringVector(
'.'.join(["LoSoTo.Steps", step, "InterpAxes"]), ['time', 'freq'])
interpMethod = parset.getString(
'.'.join(["LoSoTo.Steps", step, "InterpMethod"]), 'linear')
medAxis = parset.getString('.'.join(["LoSoTo.Steps", step, "MedAxis"]), '')
rescale = parset.getBool('.'.join(["LoSoTo.Steps", step, "Rescale"]),
False)
if interpMethod not in ["nearest", "linear", "cubic"]:
logging.error('Interpolation method must be nearest, linear or cubic.')
return 1
if rescale and medAxis == '':
logging.error('A medAxis is needed for rescaling.')
return 1
# open calibration table
css, cst = calSoltab.split('/')
cr = solFetcher(H.getSoltab(css, cst))
cAxesNames = cr.getAxesNames()
for soltab in openSoltabs(H, soltabs):
logging.info("Interpolating soltab: " + soltab._v_name)
tr = solFetcher(soltab)
tw = solWriter(soltab)
axesNames = tr.getAxesNames()
for i, interpAxis in enumerate(interpAxes[:]):
if interpAxis not in axesNames or interpAxis not in cAxesNames:
logging.error('Axis ' + interpAxis + ' not found. Ignoring.')
del interpAxes[i]
if rescale and (medAxis not in axesNames or medAxis not in cAxesNames):
logging.error('Axis ' + medAxis + ' not found. Cannot proceed.')
return 1
# axis selection
userSel = {}
for axis in tr.getAxesNames():
userSel[axis] = getParAxis(step, parset, H, axis)
tr.setSelection(**userSel)
for vals, coord, selection in tr.getValuesIter(returnAxes=interpAxes):
# construct grid
coordSel = removeKeys(coord, interpAxes)
logging.debug("Working on coords:" + str(coordSel))
# change dir if sepcified
if calDir != '':
coordSel['dir'] = calDir
cr.setSelection(**coordSel)
calValues, calCoord = cr.getValues()
# fill medAxis with the median value
if rescale:
axis = cAxesNames.index(medAxis)
calValues = np.repeat(
np.expand_dims(np.median(calValues, axis), axis),
calValues.shape[axis], axis)
# create a list of values whose coords are calPoints
calValues = np.ndarray.flatten(calValues)
# create calibrator/target coordinates arrays
calPoints = []
targetPoints = []
for interpAxis in interpAxes:
calPoints.append(calCoord[interpAxis])
targetPoints.append(coord[interpAxis])
calPoints = np.array([x for x in itertools.product(*calPoints)])
targetPoints = np.array(
[x for x in itertools.product(*targetPoints)])
# interpolation
valsNew = scipy.interpolate.griddata(calPoints, calValues,
targetPoints, interpMethod)
# fill values outside boudaries with "nearest" solutions
# NOTE: in 1D is useless due to scipy bug but this is compensated in the next if
if interpMethod != 'nearest':
valsNewNearest = scipy.interpolate.griddata(
calPoints, calValues, targetPoints, 'nearest')
# NaN is != from itself
valsNew[np.isnan(valsNew)] = valsNewNearest[np.isnan(valsNew)]
# fix bug in Scipy which put NaNs outside the convex hull in 1D for 'nearest'
if len(np.squeeze(vals).shape) == 1:
import scipy.cluster.vq as vq
valsNew = np.squeeze(valsNew)
code, dist = vq.vq(targetPoints, calPoints)
# NaN is != from itself
valsNew[np.isnan(valsNew)] = calValues[code][np.isnan(valsNew)]
valsNew = valsNew.reshape(vals.shape)
if rescale:
# rescale solutions
axis = interpAxes.index(medAxis)
valsMed = np.repeat(
np.expand_dims(np.median(vals, axis), axis),
vals.shape[axis], axis)
valsNewMed = np.repeat(
np.expand_dims(np.median(valsNew, axis), axis),
valsNew.shape[axis], axis)
valsNew = vals * valsNewMed / valsMed
#print "Rescaling by: ", valsNewMed[:,0]/valsMed[:,0]
# writing back the solutions
tw.selection = selection
tw.setValues(valsNew)
tw.addHistory('INTERP (from table %s)' % (calSoltab))
return 0
for name, st in solTabs.iteritems():
if st._v_title == 'tecscreen':
st_tec = st
if st_tec is not None:
solTypes.append('TECScreen')
solTypes = list(set(solTypes))
logging.info('Found solution types in input parmdb and H5parm: '+', '.join(solTypes))
# For each solType, select appropriate solutions and construct
# the dictionary to pass to pdb.addValues()
len_sol = {}
for solType in solTypes:
if solType != 'TECScreen':
len_sol[solType] = len(pdb.getNames(solType+':*'))
else:
tec_sf = solFetcher(st_tec)
N_times = tec_sf.getAxisLen(axis='time')
len_sol[solType] = N_times
cachedSolTabs = {}
for instrumentdbFile in instrumentdbFiles:
out_instrumentdbFile = out_globaldbFile + '/' + outroot + '_' + instrumentdbFile.split('/')[-1]
logging.info('Filling '+out_instrumentdbFile+':')
# Remove existing instrumentdb (if clobber) and create new one
if os.path.exists(out_instrumentdbFile):
if options.clobber:
shutil.rmtree(out_instrumentdbFile)
else:
logging.critical('Output instrumentdb file exists and '
'clobber = False.')
def run( step, parset, H ):
"""
subtract a clock and/or tec from a phase.
"""
import numpy as np
from losoto.h5parm import solFetcher, solWriter
soltabs = getParSoltabs( step, parset, H )
soltabsToSub = parset.getStringVector('.'.join(["LoSoTo.Steps", step, "Sub"]), [] )
ratio = parset.getBool('.'.join(["LoSoTo.Steps", step, "Ratio"]), False )
for soltab in openSoltabs( H, soltabs ):
logging.info("--> Working on soltab: "+soltab._v_name)
sf = solFetcher(soltab)
sw = solWriter(soltab, useCache = True)
sfss = [] # sol fetcher to sub tables
for soltabToSub in soltabsToSub:
ss, st = soltabToSub.split('/')
sfs = solFetcher(H.getSoltab(ss, st))
if sf.getType() != 'phase' and (sfs.getType() == 'tec' or sfs.getType() == 'clock' or sfs.getType() == 'rotationmeasure' or sfs.getType() == 'tec3rd'):
logging.warning(soltabToSub+' is of type clock/tec/rm and should be subtracted from a phase. Skipping it.')
continue
sfss.append( sfs )
logging.info('Subtracting table: '+soltabToSub)
# a major speed up if tables are assumed with same axes, check that (should be the case in almost any case)
for axisName in sfs.getAxesNames():
assert all(sfs.getAxisValues(axisName) == sf.getAxisValues(axisName))
if sf.getType() == 'phase' and (sfs.getType() == 'tec' or sfs.getType() == 'clock' or sfs.getType() == 'rotationmeasure' or sfs.getType() == 'tec3rd' ):
# the only return axes is freq, slower but better code
for vals, weights, coord, selection in sf.getValuesIter(returnAxes='freq', weight = True):
for sfs in sfss:
# restrict to have the same coordinates of phases
for i, axisName in enumerate(sfs.getAxesNames()):
sfs.selection[i] = selection[sf.getAxesNames().index(axisName)]
valsSub = np.squeeze(sfs.getValues(retAxesVals=False, weight=False))
weightsSub = np.squeeze(sfs.getValues(retAxesVals=False, weight=True))
if sfs.getType() == 'clock':
vals -= 2. * np.pi * valsSub * coord['freq']
elif sfs.getType() == 'tec':
vals -= -8.44797245e9 * valsSub / coord['freq']
elif sfs.getType() == 'tec3rd':
vals -= - 1.e21 * valsSub / np.power(coord['freq'],3)
elif sfs.getType() == 'rotationmeasure':
wav = 2.99792458e8/coord['freq']
ph = wav * wav * valsSub
if coord['pol'] == 'XX' or coord['pol'] == 'RR':
vals -= ph
elif coord['pol'] == 'YY' or coord['pol'] == 'LL':
vals += ph
else:
vals -= valsSub
# flag data that are contaminated by flagged clock/tec data
if weightsSub == 0: weights[:] = 0
sw.selection = selection
sw.setValues(vals)
sw.setValues(weights, weight = True)
else:
if ratio: sw.setValues((sf.getValues(retAxesVals=False)-sfs.getValues(retAxesVals=False))/sfs.getValues(retAxesVals=False))
else: sw.setValues(sf.getValues(retAxesVals=False)-sfs.getValues(retAxesVals=False))
weight = sf.getValues(retAxesVals=False, weight=True)
weight[sfs.getValues(retAxesVals=False, weight=True) == 0] = 0
sw.setValues(weight, weight = True)
sw.addHistory('RESIDUALS by subtracting tables '+' '.join(soltabsToSub))
sw.flush()
del sf
del sw
return 0
def run( step, parset, H ):
"""
Fits a screen to TEC values derived by the TECFIT operation.
The TEC values are read from the specified tec soltab.
The results of the fit are stored in the specified tecscreen solution table.
These values are the screen TEC values per station per pierce point per
solution interval. The pierce point locations are stored in an auxiliary
array in the output solution table.
TEC screens can be plotted with the PLOT operation by setting PlotType =
TECScreen.
The H5parm_exporter.py tool can be used to export the screen to a parmdb
that BBS and the AWimager can use. Note, however, that the output screens
are not normalized properly (any normalization was lost due to the use of
source-to-source phase gradients in the TECFIT operation). Therefore, a
direction-independent calibration must be done after exporting the screens
to a parmdb file, with the following settings in the BBS solve step:
Model.Ionosphere.Enable = T
Model.Ionosphere.Type = EXPION
"""
import numpy as np
import re
from losoto.h5parm import solFetcher, solWriter
# Switch to the Agg backend to prevent problems with pylab imports when
# DISPLAY env. variable is not set
import os
if 'DISPLAY' not in os.environ:
import matplotlib
matplotlib.use("Agg")
soltabs = getParSoltabs( step, parset, H )
outSoltabs = parset.getStringVector('.'.join(["LoSoTo.Steps", step, "OutSoltab"]), [] )
height = np.array(parset.getDoubleVector('.'.join(["LoSoTo.Steps", step, "Height"]), [200e3] ))
order = int(parset.getString('.'.join(["LoSoTo.Steps", step, "Order"]), '15' ))
# Load TEC values from TECFIT operation
indx = 0
for soltab in openSoltabs(H, soltabs):
if 'tec' not in soltab._v_title:
logging.warning('No TECFIT solution tables found for solution table '
'{0}'.format(soltabs[indx]))
continue
indx += 1
solset = soltabs[indx].split('/')[0]
logging.info('Using input solution table: {0}'.format(soltabs[indx]))
logging.info('Using output solution table: {0}'.format(outSoltabs[indx]))
# Collect station and source names and positions and times, making sure
# that they are ordered correctly.
t = solFetcher(soltab)
r, axis_vals = t.getValues()
source_names = axis_vals['dir']
source_dict = H.getSou(solset)
source_positions = []
for source in source_names:
source_positions.append(source_dict[source])
station_names = axis_vals['ant']
station_dict = H.getAnt(solset)
station_positions = []
for station in station_names:
station_positions.append(station_dict[station])
times = axis_vals['time']
# Get sizes
N_sources = len(source_names)
N_times = len(times)
N_stations = len(station_names)
N_piercepoints = N_sources * N_stations
rr = np.reshape(r.transpose([0, 2, 1]), [N_piercepoints, N_times])
heights = list(set(np.linspace(height[0], height[-1], 5)))
heights.sort()
if len(heights) > 1:
logging.info('Trying range of heights: {0} m'.format(heights))
for i, height in enumerate(heights):
# Find pierce points and airmass values for given screen height
logging.info('Using height = {0} m and order = {1}'.format(height, order))
if height < 100e3:
logging.warning("Height is less than 100e3 m.")
pp, airmass = calculate_piercepoints(np.array(station_positions),
np.array(source_positions), np.array(times), height)
# Fit a TEC screen
r_0 = 10e3
beta = 5.0 / 3.0
tec_screen, residual = fit_screen_to_tec(station_names, source_names,
pp, airmass, rr, times, height, order, r_0, beta)
total_resid = np.sum(np.abs(residual))
if i > 0:
if total_resid < best_resid:
tec_screen_best = tec_screen
pp_best = pp
height_best = height
best_resid = total_resid
else:
tec_screen_best = tec_screen
pp_best = pp
height_best = height
best_resid = total_resid
if len(heights) > 1:
logging.info('Total residual for fit: {0}\n'.format(total_resid))
# Use screen with lowest total residual
if len(heights) > 1:
tec_screen = tec_screen_best
pp = pp_best
height = height_best
logging.info('Using height (with lowest total residual) of {0} m'.format(height))
# Write the results to the output solset
dirs_out = source_names
times_out = times
ants_out = station_names
# Make output tecscreen table
outSolset = outSoltabs[indx].split('/')[0]
outSoltab = outSoltabs[indx].split('/')[1]
if not outSolset in H.getSolsets().keys():
solsetTEC = H.makeSolset(outSolset)
dirs_pos = source_positions
sourceTable = solsetTEC._f_get_child('source')
sourceTable.append(zip(*(dirs_out, dirs_pos)))
ants_pos = station_positions
antennaTable = solsetTEC._f_get_child('antenna')
antennaTable.append(zip(*(ants_out, ants_pos)))
# Store tecscreen values. The residual values are stored in the weights
# table. Flagged values of the screen have weights set to 0.0.
vals = tec_screen.transpose([1, 0, 2])
weights = residual.transpose([1, 0, 2])
tec_screen_st = H.makeSoltab(outSolset, 'tecscreen', outSoltab,
axesNames=['dir', 'time', 'ant'], axesVals=[dirs_out, times_out,
ants_out], vals=vals, weights=weights)
# Store beta, r_0, height, and order as attributes of the tecscreen
# soltab
tec_screen_st._v_attrs['beta'] = beta
tec_screen_st._v_attrs['r_0'] = r_0
tec_screen_st._v_attrs['height'] = height
tec_screen_st._v_attrs['order'] = order
# Make output piercepoint table
tec_screen_solset = tec_screen_st._v_parent._v_name
H.H.create_carray('/'+tec_screen_solset+'/'+tec_screen_st._v_name,
'piercepoint', obj=pp)
# Add histories
sw = solWriter(tec_screen_st)
sw.addHistory('CREATE (by TECSCREEN operation)')
indx += 1
return 0
def run( step, parset, H ):
from losoto.h5parm import solFetcher, solWriter
import numpy as np
import scipy.optimize
delaycomplex = lambda d, freq, y: abs(np.cos(d[0]*freq) - np.cos(y)) + abs(np.sin(d[0]*freq) - np.sin(y))
# get involved solsets using local step values or global values or all
soltabs = getParSoltabs( step, parset, H )
refAnt = parset.getString('.'.join(["LoSoTo.Steps", step, "RefAnt"]), '' )
outTab = parset.getString('.'.join(["LoSoTo.Steps", step, "OutTable"]), 'phasediff' )
maxres = parset.getFloat('.'.join(["LoSoTo.Steps", step, "MaxResidual"]), 1.)
for t, soltab in enumerate(openSoltabs( H, soltabs )):
logging.info("--> Working on soltab: "+soltab._v_name)
sf = solFetcher(soltab)
# times and ants needs to be complete or selection is much slower
times = sf.getAxisValues('time')
ants = sf.getAxisValues('ant')
# this will make a selection for the getValues() and getValuesIter()
userSel = {}
for axis in sf.getAxesNames():
userSel[axis] = getParAxis( step, parset, H, axis )
sf.setSelection(**userSel)
# some checks
solType = sf.getType()
if solType != 'phase':
logging.warning("Soltab type of "+soltab._v_name+" is of type "+solType+", should be phase. Ignoring.")
continue
if refAnt != '' and not refAnt in ants:
logging.error('Reference antenna '+refAnt+' not found.')
return 1
if refAnt == '': refAnt = ants[0]
# create new table
solsetname = soltabs[t].split('/')[0]
st = H.makeSoltab(solsetname, soltype = sf.getType(), soltab = outTab, axesNames=sf.getAxesNames(), \
axesVals=[sf.getAxisValues(axisName) for axisName in sf.getAxesNames()], \
vals=sf.getValues(retAxesVals = False), weights=sf.getValues(weight = True, retAxesVals = False), parmdbType=sf.t._v_attrs['parmdb_type'])
sw = solWriter(st)
sw.addHistory('Created by CROSSDELAY operation.')
for vals, weights, coord, selection in sf.getValuesIter(returnAxes=['freq','pol','time'], weight=True, reference = refAnt):
fitdelayguess = 1.e-10 # good guess, do not use 0 as it seems the minimizer is unstable with that
if 'RR' in coord['pol'] and 'LL' in coord['pol']:
coord1 = np.where(coord['pol'] == 'RR')[0][0]
coord2 = np.where(coord['pol'] == 'LL')[0][0]
elif 'XX' in coord['pol'] and 'YY' in coord['pol']:
coord1 = np.where(coord['pol'] == 'XX')[0][0]
coord2 = np.where(coord['pol'] == 'YY')[0][0]
if not coord['ant'] == refAnt:
logging.debug('Working on ant: '+coord['ant']+'...')
if (weights == 0.).all() == True:
logging.warning('Skipping flagged antenna: '+coord['ant'])
weights[:] = 0
else:
for t, time in enumerate(times):
# apply flags
idx = ((weights[coord1,:,t] != 0.) & (weights[coord2,:,t] != 0.))
freq = np.copy(coord['freq'])[idx]
phase1 = vals[coord1,:,t][idx]
phase2 = vals[coord2,:,t][idx]
if len(freq) < 10:
vals[:,:,t] = 0.
weights[:,:,t] = 0.
logging.debug('Not enough unflagged point for the timeslot '+str(t))
continue
if (len(idx) - len(freq))/len(freq) > 1/4.:
logging.debug('High number of filtered out data points for the timeslot '+str(t)+': '+str(len(idx) - len(freq)))
phase_diff = (phase1 - phase2)
fitresultdelay, success = scipy.optimize.leastsq(delaycomplex, [fitdelayguess], args=(freq, phase_diff))
#if t%100==0: print fitresultdelay
# fractional residual
residual = np.mean(np.abs(np.mod(fitresultdelay*freq-phase_diff + np.pi, 2.*np.pi) - np.pi))
#print "t:", t, "result:", fitresultdelay, "residual:", residual
if residual < maxres:
fitdelayguess = fitresultdelay[0]
weight = 1
else:
# high residual, flag
logging.warning('Bad solution for ant: '+coord['ant']+' (time: '+str(t)+', resdiaul: '+str(residual)+').')
weight = 0
vals[:,:,t] = 0.
vals[coord1,:,t][idx] = fitresultdelay[0]*freq/2.
vals[coord2,:,t][idx] = -1.*(fitresultdelay[0]*freq)/2.
weights[:,:,t] = 0.
weights[coord1,:,t][idx] = weight
weights[coord2,:,t][idx] = weight
# Debug plot
doplot = False
if doplot and t%500==0 and coord['ant'] == 'CS004LBA':
if not 'matplotlib' in sys.modules:
import matplotlib as mpl
mpl.rc('font',size =8 )
mpl.rc('figure.subplot',left=0.05, bottom=0.05, right=0.95, top=0.95,wspace=0.22, hspace=0.22 )
mpl.use("Agg")
import matplotlib.pyplot as plt
fig = plt.figure()
fig.subplots_adjust(wspace=0)
ax = fig.add_subplot(111)
# plot rm fit
plotdelay = lambda delay, freq: np.mod( delay*freq + np.pi, 2.*np.pi) - np.pi
ax.plot(freq, plotdelay(fitresultdelay[0], freq), "-", color='purple')
ax.plot(freq, np.mod(phase1 + np.pi, 2.*np.pi) - np.pi, 'ob' )
ax.plot(freq, np.mod(phase2 + np.pi, 2.*np.pi) - np.pi, 'og' )
ax.plot(freq, np.mod(phase_diff + np.pi, 2.*np.pi) - np.pi , '.', color='purple' )
residual = np.mod(plotdelay(fitresultdelay[0], freq)-phase_diff + np.pi,2.*np.pi)-np.pi
ax.plot(freq, residual, '.', color='yellow')
ax.set_xlabel('freq')
ax.set_ylabel('phase')
ax.set_ylim(ymin=-np.pi, ymax=np.pi)
logging.warning('Save pic: '+str(t)+'_'+coord['ant']+'.png')
plt.savefig(coord['ant']+'_'+str(t)+'.png', bbox_inches='tight')
del fig
sw.setSelection(**coord)
sw.setValues( vals )
sw.setValues( weights, weight=True )
del st
del sw
del sf
return 0
def run( step, parset, H ):
import scipy.ndimage.filters
import numpy as np
from losoto.h5parm import solFetcher, solWriter
soltabs = getParSoltabs( step, parset, H )
axesToSmooth = parset.getStringVector('.'.join(["LoSoTo.Steps", step, "Axes"]), [] )
FWHM = parset.getIntVector('.'.join(["LoSoTo.Steps", step, "FWHM"]), [] )
mode = parset.getString('.'.join(["LoSoTo.Steps", step, "Mode"]), "runningmedian" )
if mode == "runningmedian" and len(axesToSmooth) != len(FWHM):
logging.error("Axes and FWHM lenghts must be equal.")
return 1
if mode == "runningmedian":
logging.warning('Flagged data are still taken into account!')
if FWHM != [] and mode != "runningmedian":
logging.warning("FWHM makes sense only with runningmedian mode, ignoring it.")
for soltab in openSoltabs( H, soltabs ):
logging.info("Smoothing soltab: "+soltab._v_name)
sf = solFetcher(soltab)
sw = solWriter(soltab, useCache = True) # remember to flush!
# axis selection
userSel = {}
for axis in sf.getAxesNames():
userSel[axis] = getParAxis( step, parset, H, axis )
sf.setSelection(**userSel)
for i, axis in enumerate(axesToSmooth[:]):
if axis not in sf.getAxesNames():
del axesToSmooth[i]
del FWHM[i]
logging.warning('Axis \"'+axis+'\" not found. Ignoring.')
for vals, weights, coord, selection in sf.getValuesIter(returnAxes=axesToSmooth, weight=True):
if mode == 'runningmedian':
valsnew = scipy.ndimage.filters.median_filter(vals, FWHM)
elif mode == 'median':
valsnew = np.median( vals[(weights!=0)] )
elif mode == 'mean':
valsnew = np.mean( vals[(weights!=0)] )
else:
logging.error('Mode must be: runningmedian, median or mean')
return 1
sw.selection = selection
sw.setValues(valsnew)
sw.flush()
sw.addHistory('SMOOTH (over %s with mode = %s)' % (axesToSmooth, mode))
del sf
del sw
return 0
if st._v_title == 'tecscreen':
st_tec = st
if st_tec is not None:
solTypes.append('TECScreen')
solTypes = list(set(solTypes))
logging.info('Found solution types in input parmdb and H5parm: ' +
', '.join(solTypes))
# For each solType, select appropriate solutions and construct
# the dictionary to pass to pdb.addValues()
len_sol = {}
for solType in solTypes:
if solType != 'TECScreen':
len_sol[solType] = len(pdb.getNames(solType + ':*'))
else:
tec_sf = solFetcher(st_tec)
N_times = tec_sf.getAxisLen(axis='time')
len_sol[solType] = N_times
for instrumentdbFile in instrumentdbFiles:
out_instrumentdbFile = out_globaldbFile + '/' + outroot + '_' + instrumentdbFile.split(
'/')[-1]
logging.info('Filling ' + out_instrumentdbFile + ':')
# Remove existing instrumentdb (if clobber) and create new one
if os.path.exists(out_instrumentdbFile):
if options.clobber:
shutil.rmtree(out_instrumentdbFile)
else:
logging.critical('Output instrumentdb file exists and '
'clobber = False.')
def makeTECparmdb(H, solset, TECsolTab, timewidths, freq, freqwidth):
"""Returns TEC screen parmdb parameters
H - H5parm object
solset - solution set with TEC screen parameters
TECsolTab = solution table with tecscreen values
timewidths - time widths of output parmdb
freq - frequency of output parmdb
freqwidth - frequency width of output parmdb
"""
global ipbar, pbar
station_dict = H.getAnt(solset)
station_names = station_dict.keys()
station_positions = station_dict.values()
source_dict = H.getSou(solset)
source_names = source_dict.keys()
source_positions = source_dict.values()
tec_sf = solFetcher(TECsolTab)
tec_screen, axis_vals = tec_sf.getValues()
times = axis_vals['time']
beta = TECsolTab._v_attrs['beta']
r_0 = TECsolTab._v_attrs['r_0']
height = TECsolTab._v_attrs['height']
order = TECsolTab._v_attrs['order']
pp = tec_sf.t.piercepoint
N_sources = len(source_names)
N_times = len(times)
N_freqs = 1
N_stations = len(station_names)
N_piercepoints = N_sources * N_stations
freqs = freq
freqwidths = freqwidth
parms = {}
v = {}
v['times'] = times
v['timewidths'] = timewidths
v['freqs'] = freqs
v['freqwidths'] = freqwidths
for station_name in station_names:
for source_name in source_names:
v['values'] = np.zeros((N_times, N_freqs), dtype=np.double)
parmname = 'Piercepoint:X:%s:%s' % (station_name, source_name)
parms[parmname] = v.copy()
v['values'] = np.zeros((N_times, N_freqs), dtype=np.double)
parmname = 'Piercepoint:Y:%s:%s' % (station_name, source_name)
parms[parmname] = v.copy()
v['values'] = np.zeros((N_times, N_freqs), dtype=np.double)
parmname = 'Piercepoint:Z:%s:%s' % (station_name, source_name)
parms[parmname] = v.copy()
v['values'] = np.zeros((N_times, N_freqs), dtype=np.double)
parmname = 'TECfit_white:%s:%s' % (station_name, source_name)
parms[parmname] = v.copy()
v['values'] = np.zeros((N_times, N_freqs), dtype=np.double)
parmname = 'TECfit_white:0:%s:%s' % (station_name, source_name)
parms[parmname] = v.copy()
v['values'] = np.zeros((N_times, N_freqs), dtype=np.double)
parmname = 'TECfit_white:1:%s:%s' % (station_name, source_name)
parms[parmname] = v.copy()
for k in range(N_times):
D = np.resize(pp[k, :, :], (N_piercepoints, N_piercepoints, 3))
D = np.transpose(D, ( 1, 0, 2 )) - D
D2 = np.sum(D**2, axis=2)
C = -(D2 / (r_0**2))**(beta / 2.0) / 2.0
tec_fit_white = np.dot(np.linalg.inv(C),
tec_screen[:, k, :].reshape(N_piercepoints))
pp_idx = 0
for src, source_name in enumerate(source_names):
for sta, station_name in enumerate(station_names):
parmname = 'Piercepoint:X:%s:%s' % (station_name, source_name)
parms[parmname]['values'][k, 0] = pp[k, pp_idx, 0]
parmname = 'Piercepoint:Y:%s:%s' % (station_name, source_name)
parms[parmname]['values'][k, 0] = pp[k, pp_idx, 1]
parmname = 'Piercepoint:Z:%s:%s' % (station_name, source_name)
parms[parmname]['values'][k, 0] = pp[k, pp_idx, 2]
parmname = 'TECfit_white:%s:%s' % (station_name, source_name)
parms[parmname]['values'][k, 0] = tec_fit_white[pp_idx]
parmname = 'TECfit_white:0:%s:%s' % (station_name, source_name)
parms[parmname]['values'][k, 0] = tec_fit_white[pp_idx]
parmname = 'TECfit_white:1:%s:%s' % (station_name, source_name)
parms[parmname]['values'][k, 0] = tec_fit_white[pp_idx]
pp_idx += 1
pbar.update(ipbar)
ipbar += 1
time_start = times[0] - timewidths[0]/2
time_end = times[-1] + timewidths[-1]/2
v['times'] = np.array([(time_start + time_end) / 2])
v['timewidths'] = np.array([time_end - time_start])
v_r0 = v.copy()
v_r0['values'] = np.array(r_0, dtype=np.double, ndmin=2)
parms['r_0'] = v_r0
v_beta = v.copy()
v_beta['values'] = np.array(beta, dtype=np.double, ndmin=2)
parms['beta'] = v_beta
v_height = v.copy()
v_height['values'] = np.array(height, dtype=np.double, ndmin=2)
parms['height'] = v_height
return parms
def run(step, parset, H):
import scipy.ndimage.filters
import numpy as np
from losoto.h5parm import solFetcher, solWriter
soltabs = getParSoltabs(step, parset, H)
axesToSmooth = parset.getStringVector(
'.'.join(["LoSoTo.Steps", step, "Axes"]), [])
FWHM = parset.getIntVector('.'.join(["LoSoTo.Steps", step, "FWHM"]), [])
mode = parset.getString('.'.join(["LoSoTo.Steps", step, "Mode"]),
"runningmedian")
if mode == "runningmedian" and len(axesToSmooth) != len(FWHM):
logging.error("Axes and FWHM lenghts must be equal.")
return 1
if mode == "runningmedian":
logging.warning('Flagged data are still taken into account!')
if FWHM != [] and mode != "runningmedian":
logging.warning(
"FWHM makes sense only with runningmedian mode, ignoring it.")
for soltab in openSoltabs(H, soltabs):
logging.info("Smoothing soltab: " + soltab._v_name)
sf = solFetcher(soltab)
sw = solWriter(soltab, useCache=True) # remember to flush!
# axis selection
userSel = {}
for axis in sf.getAxesNames():
userSel[axis] = getParAxis(step, parset, H, axis)
sf.setSelection(**userSel)
for i, axis in enumerate(axesToSmooth[:]):
if axis not in sf.getAxesNames():
del axesToSmooth[i]
del FWHM[i]
logging.warning('Axis \"' + axis + '\" not found. Ignoring.')
for vals, weights, coord, selection in sf.getValuesIter(
returnAxes=axesToSmooth, weight=True):
if mode == 'runningmedian':
valsnew = scipy.ndimage.filters.median_filter(vals, FWHM)
elif mode == 'median':
valsnew = np.median(vals[(weights != 0)])
elif mode == 'mean':
valsnew = np.mean(vals[(weights != 0)])
else:
logging.error('Mode must be: runningmedian, median or mean')
return 1
sw.selection = selection
sw.setValues(valsnew)
sw.flush()
sw.addHistory('SMOOTH (over %s with mode = %s)' % (axesToSmooth, mode))
del sf
del sw
return 0
def run(step, parset, H):
import os
import numpy as np
from losoto.h5parm import solFetcher, solHandler
# avoids error if re-setting "agg" a second run of plot
if not 'matplotlib' in sys.modules:
import matplotlib as mpl
mpl.rc('font', size=8)
mpl.rc('figure.subplot',
left=0.05,
bottom=0.05,
right=0.95,
top=0.95,
wspace=0.22,
hspace=0.22)
mpl.use("Agg")
import matplotlib.pyplot as plt # after setting "Agg" to speed up
soltabs = getParSoltabs(step, parset, H)
minZ, maxZ = parset.getDoubleVector(
'.'.join(["LoSoTo.Steps", step, "MinMax"]), [0, 0])
prefix = parset.getString('.'.join(["LoSoTo.Steps", step, "Prefix"]), '')
# Plot various TEC-screen properties
for st_scr in openSoltabs(H, soltabs):
# Check if soltab is a tecscreen table
full_name = st_scr._v_parent._v_name + '/' + st_scr._v_name
if st_scr._v_title != 'tecscreen':
logging.warning('Solution table {0} is not a tecscreen solution '
'table. Skipping.'.format(full_name))
continue
logging.info('Using input solution table: {0}'.format(full_name))
# Plot TEC screens as images
solset = st_scr._v_parent
sf_scr = solFetcher(st_scr)
r, axis_vals = sf_scr.getValues()
source_names = axis_vals['dir']
station_names = axis_vals['ant']
station_dict = H.getAnt(solset)
station_positions = []
for station in station_names:
station_positions.append(station_dict[station])
times = axis_vals['time']
tec_screen, axis_vals = sf_scr.getValues()
times = axis_vals['time']
residuals = sf_scr.getValues(weight=True, retAxesVals=False)
height = st_scr._v_attrs['height']
order = st_scr._v_attrs['order']
beta_val = st_scr._v_attrs['beta']
r_0 = st_scr._v_attrs['r_0']
pp = sf_scr.t.piercepoint
if (minZ == 0 and maxZ == 0):
min_tec = None
max_tec = None
else:
min_tec = minZ
max_tec = maxZ
make_tec_screen_plots(pp,
tec_screen,
residuals,
np.array(station_positions),
np.array(source_names),
times,
height,
order,
beta_val,
r_0,
prefix=prefix,
remove_gradient=True,
show_source_names=False,
min_tec=min_tec,
max_tec=max_tec)
return 0
weights=vals)
logging.info("Create soltab (using default name)")
H5.makeSoltab(ss,
'amplitude',
axesNames=['axis1', 'axis2', 'axis3'],
axesVals=axesVals,
vals=vals,
weights=vals)
logging.info('Get a soltab object')
st = H5.getSoltab(ss, 'stTest')
logging.info('Get all soltabs:')
print H5.getSoltabs(ss)
print "###########################################"
logging.info('### solFetcher/solWriter - General')
Hsf = solFetcher(st)
Hsw = solWriter(st)
logging.info('Get solution Type (exp: amplitude)')
print Hsf.getType()
logging.info('Get Axes Names')
print Hsf.getAxesNames()
logging.info('Get Axis1 Len (exp: 4)')
print Hsf.getAxisLen('axis1')
logging.info('Get Axis1 Type (exp: str)')
print Hsf.getAxisType('axis1')
logging.info('Get Axis2 Type (exp: float)')
print Hsf.getAxisType('axis2')
logging.info('Get Axis1 Values (exp: a,b,c,d)')
print Hsf.getAxisValues('axis1')
logging.info('Set new axes values')
Hsw.setAxisValues('axis1', ['e', 'f', 'g', 'h'])
def run(step, parset, H):
"""
Generic unspecified step for easy expansion.
"""
import numpy as np
from losoto.h5parm import solFetcher, solWriter
# all the following are LoSoTo function to extract information from the parset
# get involved solsets using local step values or global values or all
solsets = getParSolsets(step, parset, H)
logging.info('Solset: ' + str(solsets))
# get involved soltabs using local step values or global values or all
soltabs = getParSoltabs(step, parset, H)
logging.info('Soltab: ' + str(soltabs))
# get list of SolTypes using local step values or global values or all
solTypes = getParSolTypes(step, parset, H)
logging.info('SolType: ' + str(solTypes))
# do something on every soltab (use the openSoltab LoSoTo function)
for soltab in openSoltabs(H, soltabs):
logging.info("--> Working on soltab: " + soltab._v_name)
# use the solFetcher from the H5parm lib
t = solFetcher(soltab)
tw = solWriter(soltab)
axisNames = t.getAxesNames()
logging.info("Axis names are: " + str(axisNames))
solType = t.getType()
logging.info("Soltab type is: " + solType)
# this will make a selection for the getValues() and getValuesIter()
# interpret every entry in the parset which has an axis name as a selector
userSel = {}
for axis in t.getAxesNames():
userSel[axis] = getParAxis(step, parset, H, axis)
t.setSelection(**userSel)
t.setSelection(ant=ants, pol=pols, dir=dirs)
logging.info("Selection is: " + str(t.selection))
# find axis values
logging.info("Antennas (no selection) are: " +
str(t.getAxisValues('ant', ignoreSelection=True)))
logging.info("Antennas (with selection) are: " +
str(t.getAxisValues('ant')))
# but one can also use (selection is active here!)
logging.info("Antennas (other method) are: " + str(t.ant))
logging.info("Frequencies are: " + str(t.freq))
logging.info("Directions are: " + str(t.dir))
logging.info("Polarizations are: " + str(t.pol))
# try to access a non-existent axis
t.getAxisValues('nonexistantaxis')
# now get all values given this selection
logging.info("Get data using t.val")
val = t.val
logging.debug('shape of val: ' + str(t.val.shape))
logging.info("$ val is " + str(val[0, 0, 0, 0, 100]))
weight = t.weight
time = t.time
thisTime = t.time[100]
# another way to get the data is using the getValues()
logging.info("Get data using getValues()")
grid, axes = t.getValues()
# axis names
logging.info("Axes: " + str(t.getAxesNames()))
# axis shape
print axes
print[t.getAxisLen(axis) for axis in axes] # not ordered, is a dict!
# data array shape (same of axis shape)
logging.info("Shape of values: " + str(grid.shape))
#logging.info("$ val is "+str(grid[0,0,0,0,100]))
# reset selection
t.setSelection()
logging.info('Reset selection to \'\'')
logging.info("Antennas are: " + str(t.ant))
logging.info("Frequencies are: " + str(t.freq))
logging.info("Directions are: " + str(t.dir))
logging.info("Polarizations are: " + str(t.pol))
# finally the getValuesIter allaws to iterate across all possible combinations of a set of axes
logging.info('Iteration on time/freq')
for vals, coord, selection in t.getValuesIter(
returnAxes=['time', 'freq']):
# writing back the solutions
tw.selection = selection
tw.setValues(vals)
logging.info('Iteration on time')
for vals, coord, selection in t.getValuesIter(returnAxes=['time']):
# writing back the solutions
tw.selection = selection
tw.setValues(vals)
logging.info('Iteration on dir after selection to 1 dir')
t.setSelection(dir='pointing')
for vals, coord, selection in t.getValuesIter(returnAxes=['dir']):
# writing back the solutions
tw.selection = selection
tw.setValues(vals)
return 0 # if everything went fine, otherwise 1
for name, st in solTabs.iteritems():
if st._v_title == 'tecscreen':
st_tec = st
if st_tec is not None:
solTypes.append('TECScreen')
solTypes = list(set(solTypes))
logging.info('Found solution types in input parmdb and H5parm: '+', '.join(solTypes))
# For each solType, select appropriate solutions and construct
# the dictionary to pass to pdb.addValues()
len_sol = {}
for solType in solTypes:
if solType != 'TECScreen':
len_sol[solType] = len(pdb.getNames(solType+':*'))
else:
tec_sf = solFetcher(st_tec)
N_times = tec_sf.getAxisLen(axis='time')
len_sol[solType] = N_times
for instrumentdbFile in instrumentdbFiles:
out_instrumentdbFile = out_globaldbFile + '/' + outroot + '_' + instrumentdbFile.split('/')[-1]
logging.info('Filling '+out_instrumentdbFile+':')
# Remove existing instrumentdb (if clobber) and create new one
if os.path.exists(out_instrumentdbFile):
if options.clobber:
shutil.rmtree(out_instrumentdbFile)
else:
logging.critical('Output instrumentdb file exists and '
'clobber = False.')
sys.exit(1)
def run( step, parset, H ):
import scipy.ndimage.filters
import numpy as np
from losoto.h5parm import solFetcher, solWriter
from scipy.optimize import minimize
import itertools
from scipy.interpolate import griddata
import scipy.cluster.vq as vq
def robust_std(data, sigma=3):
"""
Calculate standard deviation excluding outliers
ok with masked arrays
"""
return np.std(data[np.where(np.abs(data) < sigma * np.std(data))])
def mask_interp(vals, mask, method='nearest'):
"""
return interpolated values for masked elements
"""
this_vals = vals.copy()
#this_vals[mask] = np.interp(np.where(mask)[0], np.where(~mask)[0], vals[~mask])
#this_vals[mask] = griddata(np.where(~mask)[0], vals[~mask], np.where(mask)[0], method)
# griddata has nan bug with nearest, I need to use vq
code, dist = vq.vq(np.where(mask)[0], np.where(~mask)[0])
this_vals[ np.where(mask)[0] ] = this_vals[code]
return this_vals
tec_jump_val = 0.019628
maxsize = 300
clip = 10 # TECs over these amount of jumps are flagged
soltabs = getParSoltabs( step, parset, H )
for soltab in openSoltabs( H, soltabs ):
logging.info("Removing TEC jumps from soltab: "+soltab._v_name)
sf = solFetcher(soltab)
sw = solWriter(soltab) # remember to flush!
# TODO: check if it's a Tec table
# axis selection
userSel = {}
for axis in sf.getAxesNames():
userSel[axis] = getParAxis( step, parset, H, axis )
sf.setSelection(**userSel)
for vals, weights, coord, selection in sf.getValuesIter(returnAxes='time', weight=True):
# skip all flagged
if (weights == 0).all(): continue
# skip reference
if (np.diff(vals[(weights == 1)]) == 0).all(): continue
# kill large values
# weights[abs(vals/tec_jump_val)>clip] = 0
# interpolate flagged values to get resonable distances
# vals = mask_interp(vals, mask=(weights == 0))/tec_jump_val
# add edges to allow intervals to the borders
# vals = np.insert(vals, 0, vals[0])
# vals = np.insert(vals, len(vals), vals[-1])
vals = np.fmod(vals,tec_jump_val)
# def find_jumps(d_vals):
# # jump poistion finder
# d_smooth = scipy.ndimage.filters.median_filter( mask_interp(d_vals, mask=(abs(d_vals)>0.8)), 21 )
# d_vals -= d_smooth
# jumps = list(np.where(np.abs(d_vals) > 1.)[0])
# return [0]+jumps+[len(d_vals)-1] # add edges
#
# class Jump(object):
# def __init__(self, jumps_idx, med):
# self.idx_left = jumps_idx[0]
# self.idx_right = jumps_idx[1]
# self.jump_left = np.rint(d_vals[self.idx_left])
# self.jump_right = np.rint(d_vals[self.idx_right])
# self.size = self.idx_right-self.idx_left
# self.hight = np.median(vals[self.idx_left+1:self.idx_right+1]-med)
# if abs((self.hight-self.jump_left)-med) > abs((self.hight-self.jump_right)-med):
# self.closejump = self.jump_right
# else:
# self.closejump = self.jump_left
#
# i = 0
# while i<len(coord['time']):
# # get tec[i] - tec[i+1], i.e. the derivative assuming constant timesteps
# # this is in units of tec_jump_val!
# d_vals = np.diff(vals)
# # get jumps idx, idx=n means a jump beteen val n and n+1
# jumps_idx = find_jumps(d_vals)
#
# # get regions
# med = np.median(vals)
# jumps = [Jump(jump_idx, med) for jump_idx in zip( jumps_idx[:-1], jumps_idx[1:] )]
# jumps = [jump for jump in jumps if jump.closejump != 0]
# jumps = [jump for jump in jumps if jump.size != 0] # prevent bug on edges
# jumps = [jump for jump in jumps if jump.size < maxsize]
#
# jumps.sort(key=lambda x: (np.abs(x.size), x.hight), reverse=False) #smallest first
# #print [(j.hight, j.closejump) for j in jumps]
#
# plot = False
# if plot:
# import matplotlib.pyplot as plt
# fig, ((ax1, ax2, ax3)) = plt.subplots(3, 1, sharex=True)
# fig.subplots_adjust(hspace=0)
# d_smooth = scipy.ndimage.filters.median_filter( mask_interp(d_vals, mask=(abs(d_vals)>0.8)), 31 )
# ax1.plot(d_vals,'k-')
# ax2.plot(d_smooth,'k-')
# ax3.plot(vals, 'k-')
# [ax3.axvline(jump_idx+0.5, color='r', ls=':') for jump_idx in jumps_idx]
# ax1.set_ylabel('d_vals')
# ax2.set_ylabel('d_vals - smooth')
# ax3.set_ylabel('TEC/jump')
# ax3.set_xlabel('timestep')
# ax1.set_xlim(xmin=-10, xmax=len(d_smooth)+10)
# fig.savefig('plots/%stecjump_debug_%03i' % (coord['ant'], i))
# i+=1
#
# if len(jumps) == 0:
# break
#
# # move down the highest to the side closest to the median
# j = jumps[0]
# #print j.idx_left, j.idx_right, j.jump_left, j.jump_right, j.hight, j.closejump
#
# vals[j.idx_left+1:j.idx_right+1] -= j.closejump
# logging.debug("%s: Number of jumps left: %i - Removing jump: %i - Size %i" % (coord['ant'], len(jumps_idx)-2, j.closejump, j.size))
# re-create proper vals
# vals = vals[1:-1]*tec_jump_val
# set back to 0 the values for flagged data
vals[weights == 0] = 0
sw.selection = selection
sw.setValues(vals)
sw.setValues(weights, weight=True)
sw.addHistory('TECJUMP')
del sf
del sw
return 0
opt.add_option('-n',
'--numiter',
help='Number of iterations (default=100)',
type=int,
default=100)
(options, args) = opt.parse_args()
logger = _logging.Logger('debug')
logging = _logging.logger
n = options.numiter
solset = options.solset
h5parmFile = options.h5parm
H5 = h5parm(h5parmFile, readonly=False)
H = solFetcher(H5.getSoltab(solset, 'rotation000'))
logging.info("H5parm filename = " + h5parmFile)
parmdbFile = options.parmdb
P = lofar.parmdb.parmdb(parmdbFile)
P2 = lofar.parmdb.parmdb('tmp.parmdb', create=True)
logging.info("parmdb filename = " + parmdbFile)
######################################################
logging.info("### Read all frequencies for a pol/dir/station")
start = time.clock()
for i in range(n):
Pfreqs = P.getValuesGrid('RotationAngle:CS001LBA:3C196'
)['RotationAngle:CS001LBA:3C196']['freqs']
elapsed = (time.clock() - start)
def run( step, parset, H ):
"""
Interpolate the solutions from one table into a destination table
"""
import itertools
import scipy.interpolate
import numpy as np
from losoto.h5parm import solFetcher, solWriter
soltabs = getParSoltabs( step, parset, H )
solTypes = getParSolTypes( step, parset, H )
calSoltab = parset.getString('.'.join(["LoSoTo.Steps", step, "CalSoltab"]), '' )
calDir = parset.getString('.'.join(["LoSoTo.Steps", step, "CalDir"]), '' )
interpAxes = parset.getStringVector('.'.join(["LoSoTo.Steps", step, "InterpAxes"]), ['time','freq'] )
interpMethod = parset.getString('.'.join(["LoSoTo.Steps", step, "InterpMethod"]), 'linear' )
medAxis = parset.getString('.'.join(["LoSoTo.Steps", step, "MedAxis"]), '' )
rescale = parset.getBool('.'.join(["LoSoTo.Steps", step, "Rescale"]), False )
if interpMethod not in ["nearest", "linear", "cubic"]:
logging.error('Interpolation method must be nearest, linear or cubic.')
return 1
if rescale and medAxis == '':
logging.error('A medAxis is needed for rescaling.')
return 1
# open calibration table
css, cst = calSoltab.split('/')
cr = solFetcher(H.getSoltab(css, cst))
cAxesNames = cr.getAxesNames()
for soltab in openSoltabs( H, soltabs ):
logging.info("Interpolating soltab: "+soltab._v_name)
tr = solFetcher(soltab)
tw = solWriter(soltab)
axesNames = tr.getAxesNames()
for i, interpAxis in enumerate(interpAxes[:]):
if interpAxis not in axesNames or interpAxis not in cAxesNames:
logging.error('Axis '+interpAxis+' not found. Ignoring.')
del interpAxes[i]
if rescale and (medAxis not in axesNames or medAxis not in cAxesNames):
logging.error('Axis '+medAxis+' not found. Cannot proceed.')
return 1
# axis selection
userSel = {}
for axis in tr.getAxesNames():
userSel[axis] = getParAxis( step, parset, H, axis )
tr.setSelection(**userSel)
for vals, coord, selection in tr.getValuesIter(returnAxes=interpAxes):
# construct grid
coordSel = removeKeys(coord, interpAxes)
logging.debug("Working on coords:"+str(coordSel))
# change dir if sepcified
if calDir != '':
coordSel['dir'] = calDir
cr.setSelection(**coordSel)
calValues, calCoord = cr.getValues()
# fill medAxis with the median value
if rescale:
axis = cAxesNames.index(medAxis)
calValues = np.repeat( np.expand_dims( np.median( calValues, axis ), axis ), calValues.shape[axis], axis )
# create a list of values whose coords are calPoints
calValues = np.ndarray.flatten(calValues)
# create calibrator/target coordinates arrays
calPoints = []
targetPoints = []
for interpAxis in interpAxes:
calPoints.append(calCoord[interpAxis])
targetPoints.append(coord[interpAxis])
calPoints = np.array([x for x in itertools.product(*calPoints)])
targetPoints = np.array([x for x in itertools.product(*targetPoints)])
# interpolation
valsNew = scipy.interpolate.griddata(calPoints, calValues, targetPoints, interpMethod)
# fill values outside boudaries with "nearest" solutions
# NOTE: in 1D is useless due to scipy bug but this is compensated in the next if
if interpMethod != 'nearest':
valsNewNearest = scipy.interpolate.griddata(calPoints, calValues, targetPoints, 'nearest')
# NaN is != from itself
valsNew[ np.isnan(valsNew) ] = valsNewNearest [ np.isnan(valsNew) ]
# fix bug in Scipy which put NaNs outside the convex hull in 1D for 'nearest'
if len(np.squeeze(vals).shape) == 1:
import scipy.cluster.vq as vq
valsNew = np.squeeze(valsNew)
code, dist = vq.vq(targetPoints, calPoints)
# NaN is != from itself
valsNew[ np.isnan(valsNew) ] = calValues[code][ np.isnan(valsNew) ]
valsNew = valsNew.reshape(vals.shape)
if rescale:
# rescale solutions
axis = interpAxes.index(medAxis)
valsMed = np.repeat( np.expand_dims( np.median( vals, axis ), axis ), vals.shape[axis], axis )
valsNewMed = np.repeat( np.expand_dims( np.median( valsNew, axis ), axis ), valsNew.shape[axis], axis )
valsNew = vals*valsNewMed/valsMed
#print "Rescaling by: ", valsNewMed[:,0]/valsMed[:,0]
# writing back the solutions
tw.selection = selection
tw.setValues(valsNew)
tw.addHistory('INTERP (from table %s)' % (calSoltab))
return 0
def collect_solutions(H, dirs=None, freq_tol=1e6, solsets=None):
"""
Collects and returns phase solutions, etc. needed for fitting
Keyword arguments:
H -- H5parm object
dirs -- list of directions to use (None => all)
freq_tol -- tolerance for grouping of phase solutions into bands (Hz)
solsets -- list of solution sets in H to search (None => all)
"""
import numpy as np
from pylab import find
import re
try:
import progressbar
except ImportError:
import losoto.progressbar as progressbar
logging.info("Scanning for solutions needed for TEC fitting...")
# Determine axis lengths
sources = []
freqs = []
stations = []
if solsets is None:
solsets = H.getSolsets().keys()
N_times_max = 0
first_solset = None
for solset in solsets[:]:
logging.info(' --Scanning solution set {0}...'.format(solset))
has_phase_st = False
soltabs = H.getSoltabs(solset)
for soltab in soltabs:
# Only use soltabs with phase solutions
if 'phase' in soltabs[soltab]._v_title:
logging.info(' --Scanning solution table {0}...'.format(soltab))
has_phase_st = True
solset_sources = H.getSou(solset)
if len(solset_sources) == 1 and 'pointing' in solset_sources:
logging.info(' Found direction-independent solutions')
dir_indep = True
soln_type_dirindep = soltabs[soltab]._v_title
if first_solset is None:
first_solset = solset
else:
logging.info(' Found direction-dependent solutions')
dir_indep = False
if first_solset is None:
first_solset = solset
if not dir_indep:
soln_type_dirdep = soltabs[soltab]._v_title
logging.info(' Found sources: {0}'.format(
H.getSou(solset).keys()))
sources += H.getSou(solset).keys()
stations += H.getAnt(solset).keys()
t = solFetcher(soltabs[soltab])
solns, axis_vals = t.getValues()
times = axis_vals['time']
N_times = len(times)
freqs.append(axis_vals['freq'][0])
if N_times > N_times_max:
N_times_max = N_times
times_max = times
if not has_phase_st:
logging.info(' --No phase solutions found')
solsets.remove(solset)
# Consolidate frequencies into bands (defined by freq_tol parameter)
has_dup = True
while has_dup:
has_dup = False
for freq in freqs[:]:
nfreq = len( find( (np.array(freqs) > (freq-freq_tol)) &
(np.array(freqs) < (freq+freq_tol)) ) )
if nfreq > 1:
has_dup = True
freqs.remove(freq)
break
freqs = np.array(sorted(freqs))
N_freqs = len(freqs)
stations_set = set(stations)
sources_set = set(sources)
if 'pointing' in sources_set:
sources_set.remove('pointing')
if dirs is not None:
sources_filt = []
for dir in dirs:
if dir in sources_set:
sources_filt.append(dir)
sources_set = set(sources_filt)
source_names = np.array(sorted(list(sources_set)))
N_sources = len(set(source_names))
N_stations = len(set(stations))
N_times = N_times_max
times = times_max
logging.info('Scanning complete')
logging.info(' Number of sources: {0}'.format(N_sources))
logging.info(' Number of stations: {0}'.format(N_stations))
logging.info(' Number of times: {0}'.format(N_times))
logging.info(' Number of bands: {0}'.format(N_freqs))
if N_sources == 0 or N_stations == 0 or N_times == 0 or N_freqs == 0:
logging.error('No solutions found.')
return (None, None, None, None, None, None, None, None, None, None,
None, None)
# Initialize the arrays
freqwidths = np.zeros(N_freqs)
timewidths = np.zeros(N_times)
m = np.zeros((N_sources, N_freqs))
phases0 = np.zeros((N_sources, N_stations, N_freqs, N_times))
phases1 = np.zeros((N_sources, N_stations, N_freqs, N_times))
flags = np.ones((N_sources, N_stations, N_freqs, N_times))
source_positions = np.zeros((N_sources, 2))
solset_array_dir_dep = np.zeros((N_sources, N_freqs), dtype='|S100')
solset_array_dir_indep = np.zeros(N_freqs, dtype='|S100')
# Populate the arrays
for solset in solsets:
source_dict = H.getSou(solset)
sources = source_dict.keys()
soltabs = H.getSoltabs(solset)
for soltab in soltabs:
t = solFetcher(soltabs[soltab])
solns, axes = t.getValues()
freq = axes['freq'][0]
freq_indx = find( (np.array(freqs) > (freq-freq_tol)) &
(np.array(freqs) < (freq+freq_tol)) )
for source in sources:
if source in source_names:
source_indx = find(source_names == source)
m[source_indx, freq_indx] = 1
solset_array_dir_dep[source_indx, freq_indx] = solset
elif source=='pointing' and len(sources) == 1:
solset_array_dir_indep[freq_indx] = solset
# Collect station properties and pointing direction
station_dict = H.getAnt(first_solset)
station_names = np.array(list(stations_set))
station_positions = np.zeros((len(station_names), 3), dtype=np.float)
for i, station_name in enumerate(station_names):
station_positions[i, 0] = station_dict[station_name][0]
station_positions[i, 1] = station_dict[station_name][1]
station_positions[i, 2] = station_dict[station_name][2]
source_dict = H.getSou(first_solset)
ra = source_dict['pointing'][0]
dec = source_dict['pointing'][1]
pointing = np.array([np.cos(ra) * np.cos(dec), np.sin(ra) * np.cos(dec),
np.sin(dec)])
# Collect source positions and phase solutions for each band
logging.info('Collecting phase solutions...')
pbar = progressbar.ProgressBar(maxval=len(source_names)).start()
for i, source1 in enumerate(source_names):
for k in xrange(N_freqs):
if m[i, k]:
solset_name = str(solset_array_dir_dep[i, k])
soltab = H.getSoltab(solset=solset_name, soltab=soln_type_dirdep+'000')
solset_dir_indep_name = str(solset_array_dir_indep[k])
source_dict = H.getSou(solset_name)
source_positions[i, ] = source_dict[source1]
if solset_dir_indep_name != '':
soltab_dir_indep = H.getSoltab(solset=solset_dir_indep_name,
soltab=soln_type_dirindep+'000')
sf_dir_indep = solFetcher(soltab_dir_indep)
else:
sf_dir_indep = None
sf_dir_dep = solFetcher(soltab)
for l, station in enumerate(station_names):
if soln_type_dirdep == 'scalarphase':
sf_dir_dep.setSelection(ant=[station], dir=[source1])
else:
sf_dir_dep.setSelection(ant=[station], pol=['XX'], dir=[source1])
values_dir_dep = sf_dir_dep.getValues()
v1_phase = np.array(values_dir_dep[0]).squeeze()
times_dir_dep = values_dir_dep[1]['time']
ind = np.where(~np.isnan(v1_phase))
v1_phase = v1_phase[ind]
times_dir_dep = times_dir_dep[ind]
if sf_dir_indep is not None:
if soln_type_dirindep == 'scalarphase':
sf_dir_indep.setSelection(ant=[station])
else:
sf_dir_indep.setSelection(ant=[station], pol=['XX'])
values_dir_indep = sf_dir_indep.getValues()
v1_dir_indep = np.array(values_dir_indep[0]).squeeze()
times_dir_indep = values_dir_indep[1]['time']
ind = np.where(~np.isnan(v1_dir_indep))
v1_dir_indep = v1_dir_indep[ind]
times_dir_indep = times_dir_indep[ind]
v1_dir_indep_interp = interpolate_phase(v1_dir_indep,
times_dir_indep, times_dir_dep)
v1_phase += v1_dir_indep_interp
if len(times_dir_dep) != N_times_max:
v1_phase = interpolate_phase(v1_phase, times_dir_dep,
times_max)
phases0[i, l, k, :] = v1_phase
if soln_type_dirdep == 'scalarphase':
phases1[i, l, k, :] = v1_phase
else:
sf_dir_dep.setSelection(ant=[station], pol=['YY'], dir=[source1])
values_dir_dep = sf_dir_dep.getValues()
v1_phase = np.array(values_dir_dep[0]).squeeze()
times_dir_dep = values_dir_dep[1]['time']
ind = np.where(~np.isnan(v1_phase))
v1_phase = v1_phase[ind]
times_dir_dep = times_dir_dep[ind]
if sf_dir_indep is not None:
if soln_type_dirindep == 'scalarphase':
sf_dir_indep.setSelection(ant=[station])
else:
sf_dir_indep.setSelection(ant=[station], pol=['YY'])
values_dir_indep = sf_dir_indep.getValues()
v1_dir_indep = np.array(values_dir_indep[0]).squeeze()
times_dir_indep = values_dir_indep[1]['time']
ind = np.where(~np.isnan(v1_dir_indep))
v1_dir_indep = v1_dir_indep[ind]
times_dir_indep = times_dir_indep[ind]
v1_dir_indep_interp = interpolate_phase(v1_dir_indep,
times_dir_indep, times_dir_dep)
v1_phase += v1_dir_indep_interp
if len(times_dir_dep) != N_times_max:
v1_phase = interpolate_phase(v1_phase, times_dir_dep,
times_max)
phases1[i, l, k, :] = v1_phase
if len(times_dir_dep) != N_times_max:
# Set all weights to unity if interpolation was required
flags[i, l, k, :] = 1.0
else:
flags[i, l, k, :] = sf_dir_dep.getValues(weight=True,
retAxesVals=False)
if np.all(phases0[i, l, k, :] == 0.0):
# Check for flagged stations
flags[i, l, k, :] = 0.0
pbar.update(i)
pbar.finish()
logging.info('Collection complete')
for i, source in enumerate(source_names):
nbands = len(find(m[i, :]))
logging.info(' Source {0} has solutions in {1} bands'.format(source, nbands))
# Invert the weights to give flags (0 => unflagged, 1 => flagged)
zeroflags = np.where(flags == 0.0)
oneflags = flags.nonzero()
flags[zeroflags] = 1.0
flags[oneflags] = 0.0
return (phases0, phases1, flags, m, station_names, station_positions,
source_names, source_positions, freqs, times, pointing, soln_type_dirdep)
def run(step, parset, H):
import numpy as np
from losoto.h5parm import solFetcher, solWriter
soltabs = getParSoltabs(step, parset, H)
axesToClip = parset.getStringVector(
'.'.join(["LoSoTo.Steps", step, "Axes"]), [])
clipLevel = parset.getFloat('.'.join(["LoSoTo.Steps", step, "ClipLevel"]),
0.)
log = parset.getBool('.'.join(["LoSoTo.Steps", step, "Log"]), True)
if len(axesToClip) < 1:
logging.error("Please specify axes to clip.")
return 1
if clipLevel == 0.:
logging.error(
"Please specify factor above/below median at which to clip.")
return 1
for soltab in openSoltabs(H, soltabs):
logging.info("Clipping soltab: " + soltab._v_name)
sf = solFetcher(soltab)
# axis selection
userSel = {}
for axis in sf.getAxesNames():
userSel[axis] = getParAxis(step, parset, H, axis)
sf.setSelection(**userSel)
# some checks
for i, axis in enumerate(axesToClip[:]):
if axis not in sf.getAxesNames():
del axesToClip[i]
logging.warning('Axis \"' + axis + '\" not found. Ignoring.')
if sf.getType() != 'amplitude':
logging.error('CLIP is for "amplitude" tables, not %s.' %
sf.getType())
continue
sw = solWriter(soltab, useCache=True) # remember to flush()
before_count = 0
after_count = 0
total = 0
for vals, weights, coord, selection in sf.getValuesIter(
returnAxes=axesToClip, weight=True):
total += len(vals)
before_count += (len(weights) - np.count_nonzero(weights))
# first find the median and standard deviation
if (weights == 0).all():
valmedian = 0
else:
if log:
valmedian = np.median(np.log10(vals[(weights != 0)]))
rms = np.std(np.log10(vals[(weights != 0)]))
np.putmask(
weights,
np.abs(np.log10(vals) - valmedian) > rms * clipLevel,
0)
else:
valmedian = np.median(vals[(weights != 0)])
rms = np.std(vals[(weights != 0)])
np.putmask(weights,
np.abs(vals - valmedian) > rms * clipLevel, 0)
after_count += (len(weights) - np.count_nonzero(weights))
# writing back the solutions
sw.selection = selection
sw.setValues(weights, weight=True)
sw.addHistory('CLIP (over %s with %s sigma cut)' %
(axesToClip, clipLevel))
logging.info('Clip, flagged data: %f %% -> %f %%' \
% (100.*before_count/total, 100.*after_count/total))
sw.flush()
return 0
def run( step, parset, H ):
from losoto.h5parm import solFetcher, solWriter
soltabs = getParSoltabs( step, parset, H )
#check_parset('Axes','MaxCycles','MaxRms','Order','Replace','PreFlagZeros')
axesToFlag = parset.getStringVector('.'.join(["LoSoTo.Steps", step, "Axes"]), 'time' )
maxCycles = parset.getInt('.'.join(["LoSoTo.Steps", step, "MaxCycles"]), 5 )
maxRms = parset.getFloat('.'.join(["LoSoTo.Steps", step, "MaxRms"]), 5. )
fixRms = parset.getFloat('.'.join(["LoSoTo.Steps", step, "FixRms"]), 0 )
order = parset.getIntVector('.'.join(["LoSoTo.Steps", step, "Order"]), 3 )
replace = parset.getBool('.'.join(["LoSoTo.Steps", step, "Replace"]), False )
preflagzeros = parset.getBool('.'.join(["LoSoTo.Steps", step, "PreFlagZeros"]), False )
mode = parset.getString('.'.join(["LoSoTo.Steps", step, "Mode"]), 'smooth' )
ref = parset.getString('.'.join(["LoSoTo.Steps", step, "Reference"]), '' )
ncpu = parset.getInt('.'.join(["LoSoTo.Ncpu"]), 1 )
if ref == '': ref = None
if axesToFlag == []:
logging.error("Please specify axis to flag. It must be a single one.")
return 1
if len(axesToFlag) != len(order):
logging.error("AxesToFlag and order must be both 1 or 2 values.")
return 1
if len(order) == 1: order = order[0]
elif len(order) == 2: order = tuple(order)
mode = mode.lower()
if mode != 'smooth' and mode != 'poly' and mode != 'spline':
logging.error('Mode must be smooth, poly or spline')
return 1
for soltab in openSoltabs( H, soltabs ):
# start processes for multi-thread
mpm = multiprocManager(ncpu, flag)
logging.info("Flagging soltab: "+soltab._v_name)
sf = solFetcher(soltab)
sw = solWriter(soltab, useCache=True) # remember to flush!
# axis selection
userSel = {}
for axis in sf.getAxesNames():
userSel[axis] = getParAxis( step, parset, H, axis )
sf.setSelection(**userSel)
for axisToFlag in axesToFlag:
if axisToFlag not in sf.getAxesNames():
logging.error('Axis \"'+axis+'\" not found.')
mpm.wait()
return 1
# reorder axesToFlag as axes in the table
axesToFlag_orig = axesToFlag
axesToFlag = [coord for coord in sf.getAxesNames() if coord in axesToFlag]
if type(order) is int: order = [order]
if axesToFlag_orig != axesToFlag: order = order[::-1] # reverse order if we changed axesToFlag
solType = sf.getType()
# fill the queue (note that sf and sw cannot be put into a queue since they have file references)
for vals, weights, coord, selection in sf.getValuesIter(returnAxes=axesToFlag, weight=True, reference=ref):
mpm.put([vals, weights, coord, solType, order, mode, preflagzeros, maxCycles, fixRms, maxRms, replace, axesToFlag, selection])
#v, w, sel = flag(vals, weights, coord, solType, order, mode, preflagzeros, maxCycles, fixRms, maxRms, replace, axesToFlag, selection)
mpm.wait()
for v, w, sel in mpm.get():
sw.selection = sel
if replace:
# rewrite solutions (flagged values are overwritten)
sw.setValues(v, weight=False)
else:
sw.setValues(w, weight=True)
sw.flush()
sw.addHistory('FLAG (over %s with %s sigma cut)' % (axesToFlag, maxRms))
del sw
del sf
del soltab
return 0
def run( step, parset, H ):
import os, random
import numpy as np
from losoto.h5parm import solFetcher, solHandler
def normalize(phase):
"""
Normalize phase to the range [-pi, pi].
"""
# Convert to range [-2*pi, 2*pi].
out = np.fmod(phase, 2.0 * np.pi)
# Remove nans
np.putmask(out, out!=out, 0)
# Convert to range [-pi, pi]
out[out < -np.pi] += 2.0 * np.pi
out[out > np.pi] -= 2.0 * np.pi
return out
soltabs = getParSoltabs( step, parset, H )
# 1- or 2-element array in form X, [Y]
axesInPlot = parset.getStringVector('.'.join(["LoSoTo.Steps", step, "Axes"]), [] )
NColFig = parset.getInt('.'.join(["LoSoTo.Steps", step, "Columns"]), 0 )
figSize = parset.getIntVector('.'.join(["LoSoTo.Steps", step, "FigSize"]), [0,0] )
minZ, maxZ = parset.getDoubleVector('.'.join(["LoSoTo.Steps", step, "MinMax"]), [0,0] )
if minZ == 0: minZ = None
if maxZ == 0: maxZ = None
axisInTable = parset.getStringVector('.'.join(["LoSoTo.Steps", step, "TableAxis"]), [] )
axisInCol = parset.getStringVector('.'.join(["LoSoTo.Steps", step, "ColorAxis"]), [] )
axisInDiff = parset.getStringVector('.'.join(["LoSoTo.Steps", step, "DiffAxis"]), [] )
# log='XYZ' to set which axes to put in Log
log = parset.getString('.'.join(["LoSoTo.Steps", step, "Log"]), "" )
plotflag = parset.getBool('.'.join(["LoSoTo.Steps", step, "PlotFlag"]), False )
dounwrap = parset.getBool('.'.join(["LoSoTo.Steps", step, "Unwrap"]), False )
ref = parset.getString('.'.join(["LoSoTo.Steps", step, "Reference"]), '' )
tablesToAdd = parset.getStringVector('.'.join(["LoSoTo.Steps", step, "Add"]), [] )
makeAntPlot = parset.getBool('.'.join(["LoSoTo.Steps", step, "MakeAntPlot"]), False )
makeMovie = parset.getBool('.'.join(["LoSoTo.Steps", step, "MakeMovie"]), False )
prefix = parset.getString('.'.join(["LoSoTo.Steps", step, "Prefix"]), '' )
ncpu = parset.getInt('.'.join(["LoSoTo.Ncpu"]), 0 )
if ncpu == 0:
import multiprocessing
ncpu = multiprocessing.cpu_count()
if makeMovie:
prefix = prefix+'__tmp__'
if os.path.dirname(prefix) != '' and not os.path.exists(os.path.dirname(prefix)):
logging.debug('Creating '+os.path.dirname(prefix)+'.')
os.makedirs(os.path.dirname(prefix))
if ref == '': ref = None
sfsAdd = [ solFetcher(soltab) for soltab in openSoltabs(H, tablesToAdd) ]
for soltab in openSoltabs( H, soltabs ):
logging.info("Plotting soltab: "+soltab._v_name)
sf = solFetcher(soltab)
# axis selection
userSel = {}
for axis in sf.getAxesNames():
userSel[axis] = getParAxis( step, parset, H, axis )
sf.setSelection(**userSel)
# some checks
for axis in axesInPlot:
if axis not in sf.getAxesNames():
logging.error('Axis \"'+axis+'\" not found.')
return 1
cmesh = False
if len(axesInPlot) == 2:
cmesh = True
# not color possible in 3D
axisInCol = []
elif len(axesInPlot) != 1:
logging.error('Axes must be a len 1 or 2 array.')
return 1
if len(set(axisInTable+axesInPlot+axisInCol+axisInDiff)) != len(axisInTable+axesInPlot+axisInCol+axisInDiff):
logging.error('Axis defined multiple times.')
return 1
# just because we use lists, check that they are 1-d
if len(axisInTable) > 1 or len(axisInCol) > 1 or len(axisInDiff) > 1:
logging.error('Too many TableAxis/ColAxis/DiffAxis, they must be at most one each.')
return 1
# all axes that are not iterated by anything else
axesInFile = sf.getAxesNames()
for axis in axisInTable+axesInPlot+axisInCol+axisInDiff:
axesInFile.remove(axis)
# set subplots scheme
if axisInTable != []:
Nplots = sf.getAxisLen(axisInTable[0])
else:
Nplots = 1
# prepare antennas coord in makeAntPlot case
if makeAntPlot:
if axesInPlot != ['ant']:
logging.error('If makeAntPlot is selected the "Axes" values must be "ant"')
return 1
antCoords = [[],[]]
for ant in sf.getAxisValues('ant'): # select only user-selected antenna in proper order
antCoords[0].append(H.getAnt(sf.getAddress().split('/')[0])[ant][0])
antCoords[1].append(H.getAnt(sf.getAddress().split('/')[0])[ant][1])
else:
antCoords = []
datatype = sf.getType()
# start processes for multi-thread
mpm = multiprocManager(ncpu, plot)
# cycle on files
if makeMovie: pngs = [] # store png filenames
for vals, coord, selection in sf.getValuesIter(returnAxes=axisInDiff+axisInTable+axisInCol+axesInPlot):
# set filename
filename = ''
for axis in axesInFile:
filename += axis+str(coord[axis])+'_'
filename = filename[:-1] # remove last _
# axis vals (they are always the same, regulat arrays)
xvals = coord[axesInPlot[0]]
# if plotting antenna - convert to number
if axesInPlot[0] == 'ant':
xvals = np.arange(len(xvals))
# if plotting time - convert in h/min/s
xlabelunit=''
if axesInPlot[0] == 'time':
if xvals[-1] - xvals[0] > 3600:
xvals = (xvals-xvals[0])/3600. # hrs
xlabelunit = ' [hr]'
elif xvals[-1] - xvals[0] > 60:
xvals = (xvals-xvals[0])/60. # mins
xlabelunit = ' [min]'
else:
xvals = (xvals-xvals[0]) # sec
xlabelunit = ' [s]'
# if plotting freq convert in MHz
elif axesInPlot[0] == 'freq':
xvals = xvals/1.e6 # MHz
xlabelunit = ' [MHz]'
if cmesh:
# axis vals (they are always the same, regular arrays)
yvals = coord[axesInPlot[1]]
# same as above but for y-axis
if axesInPlot[1] == 'ant':
yvals = np.arange(len(yvals))
if len(xvals) <= 1 or len(yvals) <=1:
logging.error('3D plot must have more then one value per axes.')
mpm.wait()
return 1
ylabelunit=''
if axesInPlot[1] == 'time':
if yvals[-1] - yvals[0] > 3600:
yvals = (yvals-yvals[0])/3600. # hrs
ylabelunit = ' [hr]'
elif yvals[-1] - yvals[0] > 60:
yvals = (yvals-yvals[0])/60. # mins
ylabelunit = ' [min]'
else:
yvals = (yvals-yvals[0]) # sec
ylabelunit = ' [s]'
elif axesInPlot[1] == 'freq': # Mhz
yvals = yvals/1.e6
ylabelunit = ' [MHz]'
else:
yvals = None
ylabelunit = None
sf2 = solFetcher(soltab)
sf2.selection = selection
# cycle on tables
titles = []
dataCube = []
weightCube = []
for Ntab, (vals, coord, selection) in enumerate(sf2.getValuesIter(returnAxes=axisInDiff+axisInCol+axesInPlot)):
dataCube.append([])
weightCube.append([])
# set tile
titles.append('')
for axis in coord:
if axis in axesInFile+axesInPlot+axisInCol: continue
titles[Ntab] += axis+':'+str(coord[axis])+' '
titles[Ntab] = titles[Ntab][:-1] # remove last ' '
sf3 = solFetcher(soltab)
sf3.selection = selection
# cycle on colors
for Ncol, (vals, weight, coord, selection) in enumerate(sf3.getValuesIter(returnAxes=axisInDiff+axesInPlot, weight=True, reference=ref)):
dataCube[Ntab].append([])
weightCube[Ntab].append([])
# differential plot
if axisInDiff != []:
# find ordered list of axis
names = [axis for axis in sf.getAxesNames() if axis in axisInDiff+axesInPlot]
if axisInDiff[0] not in names:
logging.error("Axis to differentiate (%s) not found." % axisInDiff[0])
mpm.wait()
return 1
if len(coord[axisInDiff[0]]) != 2:
logging.error("Axis to differentiate (%s) has too many values, only 2 is allowed." % axisInDiff[0])
mpm.wait()
return 1
# find position of interesting axis
diff_idx = names.index(axisInDiff[0])
# roll to first place
vals = np.rollaxis(vals,diff_idx,0)
vals = vals[0] - vals[1]
weight = np.rollaxis(weight,diff_idx,0)
weight = ((weight[0]==1) & (weight[1]==1))
del coord[axisInDiff[0]]
# add tables if required (e.g. phase/tec)
for sfAdd in sfsAdd:
newCoord = {}
for axisName in coord.keys():
if axisName in sfAdd.getAxesNames():
if type(coord[axisName]) is np.ndarray:
newCoord[axisName] = coord[axisName]
else:
newCoord[axisName] = [coord[axisName]] # avoid being interpreted as regexp, faster
sfAdd.setSelection(**newCoord)
valsAdd = np.squeeze(sfAdd.getValues(retAxesVals=False, weight=False, reference=ref))
if sfAdd.getType() == 'clock':
valsAdd = 2. * np.pi * valsAdd * newCoord['freq']
elif sfAdd.getType() == 'tec':
valsAdd = -8.44797245e9 * valsAdd / newCoord['freq']
else:
logging.warning('Only Clock or TEC can be added to solutions. Ignoring: '+sfAdd.getType()+'.')
continue
# If clock/tec are single pol then duplicate it (TODO)
# There still a problem with commonscalarphase and pol-dependant clock/tec
#but there's not easy way to combine them
if not 'pol' in sfAdd.getAxesNames() and 'pol' in sf.getAxesNames():
# find pol axis positions
polAxisPos = sf.getAxesNames().key_idx('pol')
# create a new axes for the table to add and duplicate the values
valsAdd = np.addaxes(valsAdd, polAxisPos)
if valsAdd.shape != vals.shape:
logging.error('Cannot combine the table '+sfAdd.getType()+' with '+sf4.getType()+'. Wrong shape.')
mpm.wait()
return 1
vals += valsAdd
# normalize
if (sf.getType() == 'phase' or sf.getType() == 'scalarphase'):
vals = normalize(vals)
# unwrap if required
if (sf.getType() == 'phase' or sf.getType() == 'scalarphase') and dounwrap:
vals = unwrap(vals)
# is user requested axis in an order that is different from h5parm, we need to transpose
if len(axesInPlot) == 2:
if sf3.getAxesNames().index(axesInPlot[0]) < sf3.getAxesNames().index(axesInPlot[1]): vals = vals.T
dataCube[Ntab][Ncol] = np.ma.masked_array(vals, mask=(weight == 0))
# if dataCube too large (> 500 MB) do not go parallel
if np.array(dataCube).nbytes > 1024*1024*500:
logging.debug('Big plot, parallel not possible.')
plot(Nplots, NColFig, figSize, cmesh, axesInPlot, axisInTable, xvals, yvals, xlabelunit, ylabelunit, datatype, prefix+filename, titles, log, dataCube, minZ, maxZ, plotflag, makeMovie, antCoords, None)
else:
mpm.put([Nplots, NColFig, figSize, cmesh, axesInPlot, axisInTable, xvals, yvals, xlabelunit, ylabelunit, datatype, prefix+filename, titles, log, dataCube, minZ, maxZ, plotflag, makeMovie, antCoords])
if makeMovie: pngs.append(prefix+filename+'.png')
mpm.wait()
if makeMovie:
def long_substr(strings):
"""
Find longest common substring
"""
substr = ''
if len(strings) > 1 and len(strings[0]) > 0:
for i in range(len(strings[0])):
for j in range(len(strings[0])-i+1):
if j > len(substr) and all(strings[0][i:i+j] in x for x in strings):
substr = strings[0][i:i+j]
return substr
movieName = long_substr(pngs)
assert movieName != '' # need a common prefix, use prefix keyword in case
logging.info('Making movie: '+movieName)
# make every movie last 20 sec, min one second per slide
fps = np.ceil(len(pngs)/200.)
ss="mencoder -ovc lavc -lavcopts vcodec=mpeg4:vpass=1:vbitrate=6160000:mbd=2:keyint=132:v4mv:vqmin=3:lumi_mask=0.07:dark_mask=0.2:"+\
"mpeg_quant:scplx_mask=0.1:tcplx_mask=0.1:naq -mf type=png:fps="+str(fps)+" -nosound -o "+movieName.replace('__tmp__','')+".mpg mf://"+movieName+"* > mencoder.log 2>&1"
os.system(ss)
#print ss
for png in pngs: os.system('rm '+png)
return 0
def run( step, parset, H ):
"""
subtract a clock and/or tec from a phase.
"""
import numpy as np
from losoto.h5parm import solFetcher, solWriter
soltabs = getParSoltabs( step, parset, H )
soltabsToSub = parset.getStringVector('.'.join(["LoSoTo.Steps", step, "Sub"]), [] )
ratio = parset.getBool('.'.join(["LoSoTo.Steps", step, "Ratio"]), False )
for soltab in openSoltabs( H, soltabs ):
logging.info("--> Working on soltab: "+soltab._v_name)
sf = solFetcher(soltab)
sw = solWriter(soltab, useCache = True)
sfss = [] # sol fetcher to sub tables
for soltabToSub in soltabsToSub:
ss, st = soltabToSub.split('/')
sfs = solFetcher(H.getSoltab(ss, st))
if sf.getType() != 'phase' and (sfs.getType() == 'tec' or sfs.getType() == 'clock' or sfs.getType() == 'rotationmeasure' or sfs.getType() == 'tec3rd'):
logging.warning(soltabToSub+' is of type clock/tec/rm and should be subtracted from a phase. Skipping it.')
continue
sfss.append( sfs )
logging.info('Subtracting table: '+soltabToSub)
# a major speed up if tables are assumed with same axes, check that (should be the case in almost any case)
for axisName in sfs.getAxesNames():
assert all(sfs.getAxisValues(axisName) == sf.getAxisValues(axisName))
if sf.getType() == 'phase' and (sfs.getType() == 'tec' or sfs.getType() == 'clock' or sfs.getType() == 'rotationmeasure' or sfs.getType() == 'tec3rd' ):
# the only return axes is freq, slower but better code
for vals, weights, coord, selection in sf.getValuesIter(returnAxes='freq', weight = True):
for sfs in sfss:
# restrict to have the same coordinates of phases
for i, axisName in enumerate(sfs.getAxesNames()):
sfs.selection[i] = selection[sf.getAxesNames().index(axisName)]
valsSub = np.squeeze(sfs.getValues(retAxesVals=False, weight=False))
weightsSub = np.squeeze(sfs.getValues(retAxesVals=False, weight=True))
if sfs.getType() == 'clock':
vals -= 2. * np.pi * valsSub * coord['freq']
elif sfs.getType() == 'tec':
vals -= -8.44797245e9 * valsSub / coord['freq']
elif sfs.getType() == 'tec3rd':
vals -= - 1.e21 * valsSub / np.power(coord['freq'],3)
elif sfs.getType() == 'rotationmeasure':
wav = 2.99792458e8/coord['freq']
ph = wav * wav * valsSub
if coord['pol'] == 'XX':
vals += ph
elif coord['pol'] == 'YY':
vals -= ph
else:
vals -= valsSub
# flag data that are contaminated by flagged clock/tec data
if weightsSub == 0: weights[:] = 0
sw.selection = selection
sw.setValues(vals)
sw.setValues(weights, weight = True)
else:
if ratio: sw.setValues((sf.getValues(retAxesVals=False)-sfs.getValues(retAxesVals=False))/sfs.getValues(retAxesVals=False))
else: sw.setValues(sf.getValues(retAxesVals=False)-sfs.getValues(retAxesVals=False))
weight = sf.getValues(retAxesVals=False, weight=True)
weight[sfs.getValues(retAxesVals=False, weight=True) == 0] = 0
sw.setValues(weight, weight = True)
sw.addHistory('RESIDUALS by subtracting tables '+' '.join(soltabsToSub))
sw.flush()
del sf
del sw
return 0
def run( step, parset, H ):
import os, random
import numpy as np
from losoto.h5parm import solFetcher, solHandler
def normalize(phase):
"""
Normalize phase to the range [-pi, pi].
"""
# Convert to range [-2*pi, 2*pi].
out = np.fmod(phase, 2.0 * np.pi)
# Remove nans
np.putmask(out, out!=out, 0)
# Convert to range [-pi, pi]
out[out < -np.pi] += 2.0 * np.pi
out[out > np.pi] -= 2.0 * np.pi
return out
soltabs = getParSoltabs( step, parset, H )
# 1- or 2-element array in form X, [Y]
axesInPlot = parset.getStringVector('.'.join(["LoSoTo.Steps", step, "Axes"]), [] )
NColFig = parset.getInt('.'.join(["LoSoTo.Steps", step, "Columns"]), 0 )
figSize = parset.getIntVector('.'.join(["LoSoTo.Steps", step, "FigSize"]), [0,0] )
minZ, maxZ = parset.getDoubleVector('.'.join(["LoSoTo.Steps", step, "MinMax"]), [0,0] )
if minZ == 0: minZ = None
if maxZ == 0: maxZ = None
# the axis to plot on one page - e.g. ant to get all antenna's on one plot #
axisInTable = parset.getStringVector('.'.join(["LoSoTo.Steps", step, "TableAxis"]), [] )
# the axis to plot in different colours - e.g. pol to get all correlations on one plot #
axisInCol = parset.getStringVector('.'.join(["LoSoTo.Steps", step, "ColorAxis"]), [] )
# log='XYZ' to set which axes to put in Log
log = parset.getString('.'.join(["LoSoTo.Steps", step, "Log"]), "" )
plotflag = parset.getBool('.'.join(["LoSoTo.Steps", step, "PlotFlag"]), False )
dounwrap = parset.getBool('.'.join(["LoSoTo.Steps", step, "Unwrap"]), False )
ref = parset.getString('.'.join(["LoSoTo.Steps", step, "Reference"]), '' )
tablesToAdd = parset.getStringVector('.'.join(["LoSoTo.Steps", step, "Add"]), [] )
makeAntPlot = parset.getBool('.'.join(["LoSoTo.Steps", step, "MakeAntPlot"]), False )
makeMovie = parset.getBool('.'.join(["LoSoTo.Steps", step, "MakeMovie"]), False )
prefix = parset.getString('.'.join(["LoSoTo.Steps", step, "Prefix"]), '' )
ncpu = parset.getInt('.'.join(["LoSoTo.Ncpu"]), 1 )
if makeMovie:
prefix = prefix+'__tmp__'
if os.path.dirname(prefix) != '' and not os.path.exists(os.path.dirname(prefix)):
logging.debug('Creating '+os.path.dirname(prefix)+'.')
os.makedirs(os.path.dirname(prefix))
if ref == '': ref = None
sfsAdd = [ solFetcher(soltab) for soltab in openSoltabs(H, tablesToAdd) ]
# start processes for multi-thread
mpm = multiprocManager(ncpu, plot)
for soltab in openSoltabs( H, soltabs ):
logging.info("Plotting soltab: "+soltab._v_name)
sf = solFetcher(soltab)
# axis selection
userSel = {}
for axis in sf.getAxesNames():
userSel[axis] = getParAxis( step, parset, H, axis )
sf.setSelection(**userSel)
# some checks
for axis in axesInPlot:
if axis not in sf.getAxesNames():
logging.error('Axis \"'+axis+'\" not found.')
return 1
cmesh = False
if len(axesInPlot) == 2:
cmesh = True
# not color possible in 3D
axisInCol = []
elif len(axesInPlot) != 1:
logging.error('Axes must be a len 1 or 2 array.')
return 1
if len(set(axisInTable+axesInPlot+axisInCol)) != len(axisInTable+axesInPlot+axisInCol):
logging.error('Axis defined multiple times.')
return 1
if len(axisInTable) > 1 or len(axisInCol) > 1:
logging.error('Too many AxisInTable/AxisInCol, they must be at most one each.')
return 1
# all axes that are not iterated by anything else
axesInFile = sf.getAxesNames()
for axis in axisInTable+axesInPlot+axisInCol:
axesInFile.remove(axis)
# set subplots scheme
if axisInTable != []:
Nplots = sf.getAxisLen(axisInTable[0])
else:
Nplots = 1
# prepare antennas coord in makeAntPlot case
if makeAntPlot:
if axesInPlot != ['ant']:
logging.error('If makeAntPlot is selected the "Axes" values must be "ant"')
return 1
antCoords = [[],[]]
for ant in sf.getAxisValues('ant'): # select only user-selected antenna in proper order
antCoords[0].append(H.getAnt(sf.getAddress().split('/')[0])[ant][0])
antCoords[1].append(H.getAnt(sf.getAddress().split('/')[0])[ant][1])
else:
antCoords = []
datatype = sf.getType()
# cycle on files
if makeMovie: pngs = [] # store png filenames
for vals, coord, selection in sf.getValuesIter(returnAxes=axisInTable+axisInCol+axesInPlot):
# set filename
filename = ''
for axis in axesInFile:
filename += axis+str(coord[axis])+'_'
filename = filename[:-1] # remove last _
# axis vals (they are always the same, regulat arrays)
xvals = coord[axesInPlot[0]]
# if plotting antenna - convert to number
if axesInPlot[0] == 'ant':
xvals = np.arange(len(xvals))
# if plotting time - convert in h/min/s
xlabelunit=''
if axesInPlot[0] == 'time':
if len(xvals) > 3600:
xvals = (xvals-xvals[0])/3600. # hrs
xlabelunit = ' [hr]'
elif len(xvals) > 60:
xvals = (xvals-xvals[0])/60. # mins
xlabelunit = ' [min]'
else:
xvals = (xvals-xvals[0]) # sec
xlabelunit = ' [s]'
# if plotting freq convert in MHz
elif axesInPlot[0] == 'freq':
xvals = xvals/1.e6 # MHz
xlabelunit = ' [MHz]'
if cmesh:
# axis vals (they are always the same, regular arrays)
yvals = coord[axesInPlot[1]]
# same as above but for y-axis
if axesInPlot[1] == 'ant':
yvals = np.arange(len(yvals))
if len(xvals) <= 1 or len(yvals) <=1:
logging.error('3D plot must have more then one value per axes.')
return 1
ylabelunit=''
if axesInPlot[1] == 'time':
if len(yvals) > 3600:
yvals = (yvals-yvals[0])/3600. # hrs
ylabelunit = ' [hr]'
elif len(yvals) > 60:
yvals = (yvals-yvals[0])/60. # mins
ylabelunit = ' [min]'
else:
yvals = (yvals-yvals[0]) # sec
ylabelunit = ' [s]'
elif axesInPlot[1] == 'freq': # Mhz
yvals = yvals/1.e6
ylabelunit = ' [MHz]'
else:
yvals = None
ylabelunit = None
sf2 = solFetcher(soltab)
sf2.selection = selection
# cycle on tables
titles = []
dataCube = []
weightCube = []
for Ntab, (vals, coord, selection) in enumerate(sf2.getValuesIter(returnAxes=axisInCol+axesInPlot)):
dataCube.append([])
weightCube.append([])
# set tile
titles.append('')
for axis in coord:
if axis in axesInFile+axesInPlot+axisInCol: continue
titles[Ntab] += axis+':'+str(coord[axis])+' '
titles[Ntab] = titles[Ntab][:-1] # remove last ' '
sf3 = solFetcher(soltab)
sf3.selection = selection
# cycle on colors
for Ncol, (vals, weight, coord, selection) in enumerate(sf3.getValuesIter(returnAxes=axesInPlot, weight=True, reference=ref)):
dataCube[Ntab].append([])
weightCube[Ntab].append([])
# add tables if required (e.g. phase/tec)
for sfAdd in sfsAdd:
newCoord = {}
for axisName in coord.keys():
if axisName in sfAdd.getAxesNames():
if type(coord[axisName]) is np.ndarray:
newCoord[axisName] = coord[axisName]
else:
newCoord[axisName] = [coord[axisName]] # avoid being interpreted as regexp, faster
sfAdd.setSelection(**newCoord)
valsAdd = np.squeeze(sfAdd.getValues(retAxesVals=False, weight=False, reference=ref))
if sfAdd.getType() == 'clock':
valsAdd = 2. * np.pi * valsAdd * newCoord['freq']
elif sfAdd.getType() == 'tec':
valsAdd = -8.44797245e9 * valsAdd / newCoord['freq']
else:
logging.warning('Only Clock or TEC can be added to solutions. Ignoring: '+sfAdd.getType()+'.')
continue
# If clock/tec are single pol then duplicate it (TODO)
# There still a problem with commonscalarphase and pol-dependant clock/tec
#but there's not easy way to combine them
if not 'pol' in sfAdd.getAxesNames() and 'pol' in sf.getAxesNames():
# find pol axis positions
polAxisPos = sf.getAxesNames().key_idx('pol')
# create a new axes for the table to add and duplicate the values
valsAdd = np.addaxes(valsAdd, polAxisPos)
if valsAdd.shape != vals.shape:
logging.error('Cannot combine the table '+sfAdd.getType()+' with '+sf4.getType()+'. Wrong shape.')
return 1
vals += valsAdd
# normalize
if (sf.getType() == 'phase' or sf.getType() == 'scalarphase'):
vals = normalize(vals)
# unwrap if required
if (sf.getType() == 'phase' or sf.getType() == 'scalarphase') and dounwrap:
vals = unwrap(vals)
dataCube[Ntab][Ncol] = np.ma.masked_array(vals, mask=(weight == 0))
# if dataCube too large (> 500 MB) do not go parallel
if np.array(dataCube).nbytes > 1024*1024*500:
logging.debug('Big plot, parallel not possible.')
plot(Nplots, NColFig, figSize, cmesh, axesInPlot, axisInTable, xvals, yvals, xlabelunit, ylabelunit, datatype, prefix+filename, titles, log, dataCube, minZ, maxZ, plotflag, makeMovie, antCoords, None)
else:
mpm.put([Nplots, NColFig, figSize, cmesh, axesInPlot, axisInTable, xvals, yvals, xlabelunit, ylabelunit, datatype, prefix+filename, titles, log, dataCube, minZ, maxZ, plotflag, makeMovie, antCoords])
if makeMovie: pngs.append(prefix+filename+'.png')
mpm.wait()
if makeMovie:
def long_substr(strings):
"""
Find longest common substring
"""
substr = ''
if len(strings) > 1 and len(strings[0]) > 0:
for i in range(len(strings[0])):
for j in range(len(strings[0])-i+1):
if j > len(substr) and all(strings[0][i:i+j] in x for x in strings):
substr = strings[0][i:i+j]
return substr
movieName = long_substr(pngs)
assert movieName != '' # need a common prefix, use prefix keyword in case
logging.info('Making movie: '+movieName)
# make every movie last 20 sec, min one second per slide
fps = np.ceil(len(pngs)/200.)
ss="mencoder -ovc lavc -lavcopts vcodec=mpeg4:vpass=1:vbitrate=6160000:mbd=2:keyint=132:v4mv:vqmin=3:lumi_mask=0.07:dark_mask=0.2:"+\
"mpeg_quant:scplx_mask=0.1:tcplx_mask=0.1:naq -mf type=png:fps="+str(fps)+" -nosound -o "+movieName.replace('__tmp__','')+".mpg mf://"+movieName+"* > mencoder.log 2>&1"
os.system(ss)
#print ss
for png in pngs: os.system('rm '+png)
return 0