def test_swap_freq_time_axes(self):
import os, tempfile
h5fname = tempfile.mktemp(suffix='.h5')
h5 = h5parm(h5fname, readonly=False)
solset = h5.makeSolset("sol000")
dirvals = ["CasA", "VirA"]
timevals = np.arange(0, 5)
freqvals = np.arange(0, 3)
vals = np.ones((2, 3, 5))
soltab = solset.makeSoltab(soltype=b"phase",
soltabName="phase000",
axesNames=["dir", "freq", "time"],
axesVals=[dirvals, freqvals, timevals],
vals=vals,
weights=vals)
h5.close()
h5parm_swap_freq_time(h5fname)
h5 = h5parm(h5fname, readonly=False)
solset = h5.getSolset("sol000")
soltab = solset.getSoltab("phase000")
vals = soltab.getValues(retAxesVals=False)
weights = soltab.getValues(retAxesVals=False, weight=True)
axesnames = soltab.getAxesNames()
self.assertEqual(vals.shape, (2, 5, 3))
self.assertEqual(weights.shape, (2, 5, 3))
self.assertEqual(axesnames, ["dir", "time", "freq"])
os.remove(h5fname)
def main(h5in, baseline, solset='sol000', soltab='tec000', dataval=0.0):
"""
Reset solutions
Parameters
----------
h5in : str
Filename of h5parm
baseline : str
Baseline selection used in calibration. E.g.:
""[CR]*&&;!RS106HBA;!RS205HBA;!RS208HBA"
solset : str, optional
Name of solset
soltab : str, optional
Name of soltab
dataval : float, optional
Value to set solutions to
"""
h = h5parm(h5in, readonly=False)
s = h.getSolset(solset)
t = s.getSoltab(soltab)
# Make list of excluded stations and set selection to them
remotelist = [stat for stat in t.ant if '!{}'.format(stat) in baseline]
t.setSelection(ant=remotelist)
# Reset the values
reset.run(t, dataVal=float(dataval))
# Reset the weights
reweight.run(t, weightVal=1.0)
h.close()
def main(MSfile, h5parmdb, solset_name = 'calibrator', filter = '*&'):
msfile = input2strlist_nomapfile(MSfile)[0]
## reading ANTENNA table of MS
logging.info('Collecting information from the ANTENNA table of ' + str(msfile))
antennaTable = pt.table(msfile + "::ANTENNA", ack = False)
antennaNames = antennaTable.getcol('NAME')
## reading ANTENNA information of h5parm
data = h5parm(h5parmdb, readonly = True)
solset = data.getSolset(solset_name)
station_names = solset.getAnt().keys()
station_names = [station_name.decode('utf-8') for station_name in station_names]
## check whether there are more stations in the target than in the calibrator solutions
missing_stations = list(set(antennaNames) - set(station_names))
for missing_station in missing_stations:
filter += ';!' + missing_station + '*'
pass
data.close()
## return results
result = {'filter':str(filter)}
logging.info('The following stations should be used for further processing: ' + str(result['filter']))
return(result)
def plugin_main(args, **kwargs):
"""
Takes in list of targets and an h5parm solution set and returns a list of stations
in the target data which mismatch the calibrator solutions antenna table
Parameters
----------
mapfile_in : str
Mapfile for input measurement sets
h5parmdb: str
Location of the solution h5parm set
solset_name: str
Name of the solution set of the corresponding h5parm set to compare with
filter: str
Default filter constrains for the ndppp_prep_target step (usually removing International Baselines)
Returns
-------
result : dict
Output station names to filter
"""
mapfile_in = kwargs['mapfile_in']
h5parmdb = kwargs['h5parmdb']
solset_name = kwargs['solset_name']
filter = kwargs['filter']
data = DataMap.load(mapfile_in)
mslist = [data[i].file for i in xrange(len(data))]
#mslist = MSfiles.lstrip('[').rstrip(']').replace(' ','').replace("'","").split(',')
if len(mslist) == 0:
raise ValueError(
"Did not find any existing directory in input MS list!")
pass
else:
MS = mslist[0]
pass
## reading ANTENNA table of MS
antennaFile = MS + "/ANTENNA"
logging.info('Collecting information from the ANTENNA table.')
antennaTable = pt.table(antennaFile, ack=False)
antennaNames = antennaTable.getcol('NAME')
## reading ANTENNA information of h5parm
data = h5parm(h5parmdb, readonly=True)
solset = data.getSolset(solset_name)
station_names = solset.getAnt().keys()
## check whether there are more stations in the target than in the calibrator solutions
missing_stations = list(set(antennaNames) - set(station_names))
for missing_station in missing_stations:
filter += ';!' + missing_station + '*'
pass
## return results
result = {'filter': str(filter)}
return result
pass
def repoint(h5parmFile, dirname, solsetname='sol000'):
"""
rename the pointing direction of an h5parm from 'pointing' to 'dirname'
"""
dirname = '%s' % dirname
# open h5parm
h5 = h5parm(h5parmFile, readonly=False)
ss = h5.getSolset(solsetname)
# rename each axes (must re-create the array as otherwise it truncates the dir name to the previous max string length)
for tab in ss.getSoltabs():
if 'dir' in tab.getAxesNames():
tab.obj._v_file.remove_node('/' + tab.getAddress(), 'dir')
tab.obj._v_file.create_array('/' + tab.getAddress(),
'dir',
obj=[dirname.encode()])
# rename directions table
sourceTable = ss.obj.source
direction = sourceTable[0][1]
logger.info('%s: update dir name "%s" -> "%s".' %
(h5parmFile, sourceTable[0][0], dirname))
sourceTable[0] = (dirname, direction)
# write h5parm
sourceTable.close()
h5.close()
def fixbeam_ST001(H5name):
""" Replace beam corrections for the phased up station ST001 with correct corrections.
Args:
H5name (str): path to the h5parm containing the corrections.
Returns:
None
"""
H5 = h5parm.h5parm(H5name, readonly=False)
ants = H5.getSolset('sol000').getAnt().keys()
antsrs = fnmatch.filter(ants, 'RS*')
if 'ST001' in ants:
amps = H5.getSolset('sol000').getSoltab('amplitude000').getValues()
ampvals = H5.getSolset('sol000').getSoltab(
'amplitude000').getValues()[0]
phasevals = H5.getSolset('sol000').getSoltab('phase000').getValues()[0]
idx = np.where(amps[1]['ant'] == 'ST001')[0][0]
idxrs = np.where(amps[1]['ant'] == antsrs[0])[0][0]
ampvals[:, :, idx, 0, :] = ampvals[:, :, idxrs, 0, :]
phasevals[:, :, idx, 0, :] = 0.0
H5.getSolset('sol000').getSoltab('amplitude000').setValues(ampvals)
H5.getSolset('sol000').getSoltab('phase000').setValues(phasevals)
H5.close()
def loop3_difmap(infile, datacolumn='CORRECTED_DATA', startmod=True):
os.system('rm CORPLT')
fitsfile = dif_script(infile, 'XX', startmod=startmod)
ampXX, amperrXX, phsXX, phserrXX, utXX, stnXX = corplt2array()
os.system('mv CORPLT CORPLT_XX')
fitsfile = dif_script(infile, 'YY', startmod=startmod)
ampYY, amperrYY, phsYY, phserrYY, utYY, stnYY = corplt2array()
os.system('mv CORPLT CORPLT_YY')
amp = np.rollaxis(np.dstack((ampXX, ampYY)), 2, 0)
amperr = np.rollaxis(np.dstack((amperrXX, amperrYY)), 2, 0)
phs = np.rollaxis(np.dstack((phsXX, phsYY)), 2, 0)
phserr = np.rollaxis(np.dstack((phserrXX, phserrYY)), 2, 0)
ut, stn = utXX, stnXX
h5parmfile = fitsfile.replace('.fits', '_auto.h5')
data = h5parm(h5parmfile, readonly=False)
outSolset = data.makeSolset('sol000')
outSolset.makeSoltab(soltype='amplitude',
soltabName='amplitude000',
axesNames=['pol', 'ant', 'time'],
axesVals=[['XX', 'YY'], stn, ut],
vals=amp,
weights=np.ones_like(amp))
outSolset.makeSoltab(soltype='phase',
soltabName='phase000',
axesNames=['pol', 'ant', 'time'],
axesVals=[['XX', 'YY'], stn, ut],
vals=phs,
weights=np.ones_like(phs))
data.close()
def main(h5parmfile,
solsetname='sol000',
phasesoltabname='phase000',
ampsoltabname='amplitude000'):
"""
Flag residual gains
Parameters
----------
h5parmfile : str
Filename of h5parm
phasesoltabname : str, optional
Name of phase soltab
ampsoltabname : str, optional
Name of amplitude soltab
"""
# Read in solutions
H = h5parm(h5parmfile, readonly=False)
solset = H.getSolset(solsetname)
phsoltab = solset.getSoltab(phasesoltabname)
ampsoltab = solset.getSoltab(ampsoltabname)
# Flag
operations.flagstation.run(phsoltab, 'resid', nSigma=3.0)
operations.flagstation.run(ampsoltab, 'resid', nSigma=5.0)
# Reset phases to 0 and amps to 1, so only the new flags are applied
operations.reset.run(phsoltab)
operations.reset.run(ampsoltab)
def losotolofarbeam(parmdb,
soltabname,
ms,
inverse=False,
useElementResponse=True,
useArrayFactor=True,
useChanFreq=True):
""" Do the beam correction via this imported losoto operation
Args:
parmdb (str): name of the h5parm to work on.
soltabname (str): name of the soltab to operate on.
inverse (bool): apply the inverse beam correction.
useElementResponse (bool): apply the element beam correction.
useArrayFactor (bool): apply the array factor correction.
useChanFreq (bool): operate per channel.
"""
H5 = h5parm.h5parm(parmdb, readonly=False)
soltab = H5.getSolset('sol000').getSoltab(soltabname)
sr = stationresponse(ms, inverse, useElementResponse, useArrayFactor,
useChanFreq)
numants = pt.taql('select gcount(*) as numants from ' + ms +
'::ANTENNA').getcol('numants')[0]
times = soltab.getAxisValues('time')
for vals, coord, selection in soltab.getValuesIter(
returnAxes=['ant', 'time', 'pol', 'freq'], weight=False):
vals = losoto.lib_operations.reorderAxes(
vals, soltab.getAxesNames(), ['ant', 'time', 'freq', 'pol'])
for stationnum in range(numants):
logging.debug('Working on station number %i' % stationnum)
for itime, time in enumerate(times):
beam = sr.evaluateStation(time=time, station=stationnum)
# Reshape from [nfreq, 2, 2] to [nfreq, 4]
beam = beam.reshape(beam.shape[0], 4)
if soltab.getAxisLen('pol') == 2:
beam = beam[:, [0, 3]] # get only XX and YY
if soltab.getType() == 'amplitude':
vals[stationnum, itime, :, :] = np.abs(beam)
elif soltab.getType() == 'phase':
vals[stationnum, itime, :, :] = np.angle(beam)
else:
logging.error(
'Beam prediction works only for amplitude/phase solution tables.'
)
return 1
vals = losoto.lib_operations.reorderAxes(
vals, ['ant', 'time', 'freq', 'pol'], [
ax for ax in soltab.getAxesNames()
if ax in ['ant', 'time', 'freq', 'pol']
])
soltab.setValues(vals, selection)
H5.close()
def get_values(h5, station='ST001', polarisation='XX'):
station_exists = False
h = lh5.h5parm(h5, readonly=True)
phase = h.getSolset('sol000').getSoltab('phase000')
time = phase.time
for ant in range(len(phase.ant)):
if phase.ant[ant] == station:
station_exists = True
break
if not station_exists:
print('{} does not exist in {}.'.format(station, h5))
h.close()
sys.exit()
try:
reordered_values = reorderAxes(phase.val, phase.getAxesNames(),
['time', 'freq', 'ant', 'pol', 'dir'])
my_values_xx = reordered_values[:, 0, ant, 0, 0]
my_values_yy = reordered_values[:, 0, ant, 1, 0]
except:
reordered_values = reorderAxes(phase.val, phase.getAxesNames(),
['time', 'freq', 'ant', 'pol'])
my_values_xx = reordered_values[:, 0, ant, 0]
my_values_yy = reordered_values[:, 0, ant, 1]
h.close()
if polarisation == 'XX':
values = my_values_xx
elif polarisation == 'YY':
values = my_values_yy
return values, time
def plugin_main(args, **kwargs):
fileid = kwargs['mapfile_in']
datamap = DataMap.load(fileid)
hdf5File = os.path.join(kwargs['hdf5_dir'],kwargs['hdf5file'])
if kwargs.has_key('instrument'):
instrument = kwargs['instrument']
else:
instrument = '/instrument'
if kwargs.has_key('compression'):
compression = int(kwargs['compression'])
else:
compression = 5
if kwargs.has_key('solset'):
solsetName = kwargs['solset']
else:
solsetName = None
# Check is all the necessary files are available
antennaFile = os.path.join(datamap[0].file,'ANTENNA')
if not os.path.isdir(antennaFile):
logging.critical('Missing ANTENNA table.')
sys.exit(1)
fieldFile = os.path.join(datamap[0].file,'FIELD')
if not os.path.isdir(fieldFile):
logging.critical('Missing FIELD table.')
sys.exit(1)
skydbFile = os.path.join(datamap[0].file,'sky')
if not os.path.isdir(skydbFile):
logging.critical('Missing sky table.')
sys.exit(1)
#generate list of parmDB-filenames
parmDBnames = [ MS.file+instrument for MS in datamap ]
#create and fill the hdf5-file:
solset = parmDBs2h5parm(hdf5File, parmDBnames, antennaFile, fieldFile, skydbFile, compression=compression, solsetName=solsetName)
# Add CREATE entry to history
h5parmDB = h5parm(hdf5File, readonly = False)
soltabs = h5parmDB.getSoltabs(solset=solset)
for st in soltabs:
sw = solWriter(soltabs[st])
sw.addHistory('CREATE (by PipelineStep_losotoImporter from %s / %s - %s)' % (os.path.abspath(''),
os.path.basename(parmDBnames[0]), os.path.basename(parmDBnames[-1]) ) )
h5parmDB.close()
#generate mapfile and wrap up
mapfileentry = {}
mapfileentry['host'] = 'localhost'
mapfileentry['file'] = hdf5File
mapfileentry['skip'] = False
outfileid = os.path.join(kwargs['mapfile_dir'], kwargs['filename'])
outmap = open(outfileid, 'w')
outmap.write(repr([mapfileentry]))
outmap.close()
result = {}
result['mapfile'] = outfileid
return result
def h5_values(my_h5parm, station):
lo = lh5.h5parm(my_h5parm, readonly = False)
phase = lo.getSolset('sol000').getSoltab('phase000')
values = phase.val[0, 0, station, 0, :] # (pol, dir, ant, freq, time)
lo.close()
return values
def main(losotoname, store_basename, refstationID=2, sourceID=0):
# station-ID 2 is more likely to be on the superterp
# I don't imagine someone would use a sourceID != 0
inh5parm = h5parm(losotoname ,readonly=True)
phasetab = inh5parm.getSoltab('sol000','phase000')
phases_tmp = np.copy(phasetab.val)
freqs = np.copy(phasetab.freq)
stationsnames = [ stat for stat in phasetab.ant]
# this gets the subband number to any given frequency in HBA-low
subbands = np.unique(np.round(freqs/195.3125e3-512.))
nsubbands = len(subbands)
nchan = len(freqs)/nsubbands
if nsubbands*nchan != len(freqs):
print "find_cal_global_phaseoffset_losoto.py: irregular number of ch/SB detected! Bailing out!"
print " nchan %d, nSB: %d, nfreq: %d" % (nchan, nsubbands, len(freqs))
sys.exit(1)
tmpfreqs = freqs.reshape([nsubbands,nchan])
freq_per_sb = np.mean(tmpfreqs,axis=1)
nstations = len(stationsnames)
refphases = phases_tmp[:,sourceID,refstationID,:,:]
# loop over all stations:
for istat in xrange(nstations):
phases_00 = phases_tmp[0,sourceID,istat,:,:]-refphases[0,:,:]
phases_11 = phases_tmp[1,sourceID,istat,:,:]-refphases[1,:,:]
phases_diff = normalize(phases_00-phases_11)
tmp_phases_diff = np.median(phases_diff,axis=1)
med_phases_diff = np.median(tmp_phases_diff.reshape([nsubbands,nchan]),axis=1)
if istat == 0:
global_stat_offsets = med_phases_diff
else:
global_stat_offsets = np.vstack( (global_stat_offsets, med_phases_diff) )
global_stat_offsets_smoothed = np.zeros([nsubbands,nstations])
for istat in xrange(nstations):
global_stat_offsets_smoothed[:,istat] = s.medfilt(global_stat_offsets[istat,:], kernel_size=15)
np.save('freqs_for_phase_array.npy', freq_per_sb)
np.save(store_basename.strip() + '_phase_array.npy', global_stat_offsets_smoothed)
np.save(store_basename.strip() + '_station_names.npy', stationsnames)
# do the plotting!
Nr = int(np.sqrt(nstations))
if Nr*(Nr+1) > nstations:
Nc = Nr+1
else:
Nc = Nr+1
Nr = Nc
f, axs = pl.subplots(Nr, Nc, sharex=True, sharey=True, figsize=(16,12))
ax = axs.reshape((Nr*Nc,1))
for istat, stat in enumerate(stationsnames):
ax[istat][0].set_title(stat)
ax[istat][0].plot(subbands, global_stat_offsets[istat,:],'b')
ax[istat][0].plot(subbands, global_stat_offsets_smoothed[:,istat],'g')
ax[istat][0].set_ylim(-3.2,3.2)
ax[istat][0].set_xlim(np.min(subbands), np.max(subbands))
f.savefig('phase_xx_yy_offset.png')
def process_diag(fname):
H5 = h5parm.h5parm(fname,readonly=False)
stb = H5.getSolset('sol000').getSoltab('amplitude000').getValues()
stb_shape = stb[0].shape
num_freqs = stb_shape[1]
for freq in range(num_freqs):
print(np.nanmean(stb[0][:,freq,:,:,:]))
mean_amp = np.nanmean(stb[0][:,freq,:,:,:])
stb[0][:,freq,:,:,:] *= 1/mean_amp
H5.getSolset('sol000').getSoltab('amplitude000').setValues(stb[0])
H5.close()
def main(msfileslist,
hdf5fileName,
hdf5_dir='.',
instrument='/instrument',
solsetName=None,
compression=5):
tmp_msfiles = msfileslist.lstrip('[').rstrip(']').split(',')
msfiles = [MS.strip("\' \"") for MS in tmp_msfiles]
hdf5File = os.path.join(hdf5_dir, hdf5fileName)
compression = int(compression) #doesn't hurt if it is already an int
instrument = instrument.strip()
# Check is all the necessary files are available
antennaFile = os.path.join(msfiles[0], 'ANTENNA')
if not os.path.isdir(antennaFile):
logging.critical('Missing ANTENNA table.')
sys.exit(1)
fieldFile = os.path.join(msfiles[0], 'FIELD')
if not os.path.isdir(fieldFile):
logging.critical('Missing FIELD table.')
sys.exit(1)
skydbFile = os.path.join(msfiles[0], 'sky')
if not os.path.isdir(skydbFile):
logging.warning(
'No sky table found. (Direction-dependent parameters will not work.)'
)
skydbFile = None
#generate list of parmDB-filenames
parmDBnames = [MS + instrument for MS in msfiles]
#create and fill the hdf5-file:
solset = parmDBs2h5parm(hdf5File,
parmDBnames,
antennaFile,
fieldFile,
skydbFile,
compression=compression,
solsetName=solsetName)
# Add CREATE entry to history
h5parmDB = h5parm(hdf5File, readonly=False)
soltabs = h5parmDB.getSoltabs(solset=solset)
for st in soltabs:
sw = solWriter(soltabs[st])
sw.addHistory('CREATE (by losotoImporter from %s / %s - %s)' %
(os.path.abspath(''), os.path.basename(
parmDBnames[0]), os.path.basename(parmDBnames[-1])))
h5parmDB.close()
result = {}
result['h5parm'] = hdf5File
return result
def main(h5parmdb, solsetName='sol000', pointing='POINTING'):
data = h5parm(h5parmdb, readonly=False)
solset = data.getSolset(solsetName)
sources = solset.obj._f_get_child('source')
direction = list(sources[0][-1])
for i in numpy.arange(len(sources)):
sources[i] = (pointing, direction)
data.close()
return (0)
def main(h5parm1, h5parm2, outh5parm, solset1='sol000', solset2='sol000'):
"""
Combines two h5parms
Parameters
----------
h5parm1 : str
Filenames of fast-phase h5parm
h5parm2 : str
Filenames of slow-gain h5parm
outh5parm : str
Filename of the output h5parm
solset1 : str, optional
Name of solset for h5parm1
solset2 : str, optional
Name of solset for h5parm2
"""
h1 = h5parm(h5parm1, readonly=False)
h2 = h5parm(h5parm2, readonly=False)
ss1 = h1.getSolset(solset=solset1)
ss2 = h2.getSolset(solset=solset2)
# Rename slow-phase soltab before combining to avoid conflict with fast-phase soltab
if 'phase000' in ss2.getSoltabNames():
soltab = ss2.getSoltab('phase000')
soltab.rename('phase001')
if os.path.exists(outh5parm):
os.remove(outh5parm)
ho = h5parm(outh5parm, readonly=False)
sso = ho.makeSolset(solsetName='sol000', addTables=False)
ss1.obj._f_copy_children(sso.obj, recursive=True, overwrite=True)
ss2.obj._f_copy_children(sso.obj, recursive=True, overwrite=True)
h1.close()
h2.close()
ho.close()
def main(h5parm1, h5parm2, outh5parm, solset1='sol000', solset2='sol000'):
"""
Combines two h5parms
Parameters
----------
h5parm1 : str
Filenames of fast-phase h5parm
h5parm2 : str
Filenames of slow-gain h5parm
outh5parm : str
Filename of the output h5parm
solset1 : str, optional
Name of solset for h5parm1
solset2 : str, optional
Name of solset for h5parm2
"""
h1 = h5parm(h5parm1, readonly=False)
h2 = h5parm(h5parm2, readonly=False)
ss1 = h1.getSolset(solset=solset1)
ss2 = h2.getSolset(solset=solset2)
# Rename slow-phase soltab before combining to avoid conflict with fast-phase soltab
if 'phase000' in ss2.getSoltabNames():
soltab = ss2.getSoltab('phase000')
soltab.rename('phase001')
if os.path.exists(outh5parm):
os.remove(outh5parm)
ho = h5parm(outh5parm, readonly=False)
sso = ho.makeSolset(solsetName='sol000', addTables=False)
ss1.obj._f_copy_children(sso.obj, recursive=True, overwrite=True)
ss2.obj._f_copy_children(sso.obj, recursive=True, overwrite=True)
h1.close()
h2.close()
ho.close()
def get_prefactor_freqs( solname='solutions.h5', solset='target' ):
sols = h5parm( solname )
ss = sols.getSolset( solset )
st_names = ss.getSoltabNames()
ph_sol_name = [ xx for xx in st_names if 'extract' not in xx ][0]
st = ss.getSoltab(ph_sol_name)
freqs = st.getAxisValues('freq')
freqstep = 1953125.0 ## the value for 10 subbands
f_arr = []
for xx in range(len(freqs)):
fmin = freqs[xx] - freqstep/2.
fmax = freqs[xx] + freqstep/2.
f_arr.append(np.array([fmin,fmax]))
return( f_arr )
def fit(self):
"""
Fitting is not needed: the input solutions are used directly, after
referencing the phases to a single station
"""
# Open solution tables
H = h5parm(self.input_h5parm_filename)
solset = H.getSolset(self.input_solset_name)
soltab_ph = solset.getSoltab(self.input_phase_soltab_name)
if not self.phase_only:
soltab_amp = solset.getSoltab(self.input_amplitude_soltab_name)
# Input data are [time, freq, ant, dir, pol] for slow amplitudes
# and [time, freq, ant, dir] for fast phases (scalarphase). We reference
# the phases to the station with the least amount of flagged solutions,
# drawn from the first 10 stations (to ensure it is fairly central)
self.vals_ph = soltab_ph.val
ref_ind = misc.get_reference_station(soltab_ph, 10)
vals_ph_ref = self.vals_ph[:, :, ref_ind, :].copy()
for i in range(len(soltab_ph.ant)):
# Subtract phases of reference station
self.vals_ph[:, :, i, :] -= vals_ph_ref
self.times_ph = soltab_ph.time
self.freqs_ph = soltab_ph.freq
if not self.phase_only:
self.log_amps = False
self.vals_amp = soltab_amp.val
self.times_amp = soltab_amp.time
self.freqs_amp = soltab_amp.freq
else:
self.vals_amp = np.ones_like(self.vals_ph)
self.times_amp = self.times_ph
self.freqs_amp = self.freqs_ph
self.source_names = soltab_ph.dir
self.source_dict = solset.getSou()
self.source_positions = []
for source in self.source_names:
self.source_positions.append(self.source_dict[source])
self.station_names = soltab_ph.ant
self.station_dict = solset.getAnt()
self.station_positions = []
for station in self.station_names:
self.station_positions.append(self.station_dict[station])
def fix_h5(self, parmname, also_amp = False):
'''
THIS FUNCTION IS MISBEHAVING
'''
H5 = h5parm.h5parm(self.ms + parmname, readonly = False)
phases = H5.getSolset('sol000').getSoltab('phase000').getValues()
phasevals = phases[0]
antennas = phases[1]['ant']
# Select ids of international, remote and both
idx_rem_int = np.where([not 'CS' in ant for ant in antennas])
idx_int = np.where([not ('CS' in ant or 'RS' in ant) for ant in antennas])
idx_rem = np.where(['RS' in ant for ant in antennas])
phasevals[:,:,idx_rem_int,:,:] = 0.0 # Try setting all directions?
if also_amp:
ampvals = H5.getSolset('sol000').getSoltab('amplitude000').getValues()[0]
ampvals[:,:,idx_rem,0,:] = 1.0
ampvals[:,:,idx_int,0,:] = 2.0
H5.getSolset('sol000').getSoltab('amplitude000').setValues(ampvals)
H5.getSolset('sol000').getSoltab('phase000').setValues(phasevals)
H5.close()
def main(msfileslist, hdf5fileName, hdf5_dir='.', instrument='/instrument', solsetName=None, compression=5):
tmp_msfiles = msfileslist.lstrip('[').rstrip(']').split(',')
msfiles = [ MS.strip("\' \"") for MS in tmp_msfiles]
hdf5File = os.path.join(hdf5_dir,hdf5fileName)
compression = int(compression) #doesn't hurt if it is already an int
instrument = instrument.strip()
# Check is all the necessary files are available
antennaFile = os.path.join(msfiles[0],'ANTENNA')
if not os.path.isdir(antennaFile):
logging.critical('Missing ANTENNA table.')
sys.exit(1)
fieldFile = os.path.join(msfiles[0],'FIELD')
if not os.path.isdir(fieldFile):
logging.critical('Missing FIELD table.')
sys.exit(1)
skydbFile = os.path.join(msfiles[0],'sky')
if not os.path.isdir(skydbFile):
logging.critical('Missing sky table.')
sys.exit(1)
#generate list of parmDB-filenames
parmDBnames = [ MS+instrument for MS in msfiles ]
#create and fill the hdf5-file:
solset = parmDBs2h5parm(hdf5File, parmDBnames, antennaFile, fieldFile, skydbFile, compression=compression, solsetName=solsetName)
# Add CREATE entry to history
h5parmDB = h5parm(hdf5File, readonly = False)
soltabs = h5parmDB.getSoltabs(solset=solset)
for st in soltabs:
sw = solWriter(soltabs[st])
sw.addHistory('CREATE (by losotoImporter from %s / %s - %s)' % (os.path.abspath(''),
os.path.basename(parmDBnames[0]), os.path.basename(parmDBnames[-1]) ) )
h5parmDB.close()
result = {}
result['h5parm'] = hdf5File
return result
def addpol(h5parmFile, soltabname, solsetname='sol000'):
"""
add pol axes on a soltab
"""
# open h5parm
logger.info('%s: add pol axis to %s.' % (h5parmFile, soltabname))
h5 = h5parm(h5parmFile, readonly=False)
ss = h5.getSolset(solsetname)
st = ss.getSoltab(soltabname)
if 'pol' in st.getAxesNames():
h5.close()
logger.warning('%s: polarisation axis already present in %s.' %
(h5parmFile, soltabname))
return
# create values for new soltab
typ = st.getType()
axesNames = st.getAxesNames() + ['pol']
axesVals = [st.getAxisValues(axisName)
for axisName in st.getAxesNames()] + [np.array(['XX', 'YY'])]
vals = st.getValues(retAxesVals=False)
vals = np.array([vals, vals])
vals = np.moveaxis(vals, 0, -1)
weights = st.getValues(weight=True, retAxesVals=False)
weights = np.array([weights, weights])
weights = np.moveaxis(weights, 0, -1)
# remove old soltab
st.delete()
# make new soltab
soltabout = ss.makeSoltab(soltype = typ, soltabName = soltabname, axesNames=axesNames, \
axesVals=axesVals, vals=vals, weights=weights)
# write h5parm
h5.close()
def h5parm_swap_freq_time(h5parmname, solset='sol000', soltab='all'):
"""Open an H5Parm and swap the frequency and time axes to make the frequency the fastest varying axis
Parameters
----------
h5parmname : Soltab
H5Parm filename
solset : str
Solset name (default 'sol000')
soltab : str
Soltab name or 'all' for all soltabs
"""
h5 = h5parm(h5parmname, False)
solset = h5.getSolset('sol000')
if soltab=='all':
soltabnames = solset.getSoltabNames()
else:
soltabnames = [soltab]
for soltabname in soltabnames:
soltab = solset.getSoltab(soltabname)
soltab_swap_freq_time(soltab)
def main(h5file, refstat=None):
"""
Plot solutions vs. time
Parameters
----------
h5file : str
Name of solution h5parm file
refstat : str, optional
Name of referance station. If None, the first stations is used
"""
h = h5parm(h5file)
ss = h.getSolset('sol000')
sols = ['phase000', 'amplitude001', 'phase001']
roots = ['tec+scalarphase_', 'slow-amplitude_', 'slow-phase_']
ncols = [1, 0, 0]
colors = ['', 'pol', 'pol']
minmaxes = [[-3.2, 3.2], [0, 0], [-3.2, 3.2]]
for sol, root, ncol, color, minmax in zip(sols, roots, ncols, colors, minmaxes):
st = ss.getSoltab(sol)
if sol == 'phase000':
# add tec000
stadd = ['tec000']
else:
stadd = ''
if refstat is not None:
ref = refstat
else:
ref = st.ant[0]
if sol == 'amplitude001':
ref = ''
print('Plotting {} solutions...'.format(root[:-1]))
plot.run(st, ['time'], axisInTable='ant', axisInCol=color, NColFig=ncol, refAnt=ref,
prefix=root, minmax=minmax, soltabsToAdd=stadd)
h.close()
def main():
my_h5 = '/data5/sean/hba/3c273-tests/concatenate.gaincal.solutions.h5'
my_solset = 'sol000'
my_solutions = ['amplitude'] # , 'phase']
my_polarisations = ['XX', 'YY']
h = h5.h5parm(my_h5)
solset = h.getSolset(my_solset)
for solutions in my_solutions:
my_soltab = solutions + '000'
soltab = solset.getSoltab(my_soltab)
for i, ant in enumerate(soltab.ant):
for polname in my_polarisations:
plot(solns=solutions,
values=soltab.val,
ant=ant,
idx=i,
polname=polname)
# sys.exit()
h.close()
def h5parm_swap_freq_time(h5parmname, solset='sol000', soltab='all'):
"""Open an H5Parm and swap the frequency and time axes to make the frequency the fastest varying axis
Parameters
----------
h5parmname : Soltab
H5Parm filename
solset : str
Solset name (default 'sol000')
soltab : str
Soltab name or 'all' for all soltabs
"""
h5 = h5parm(h5parmname, False)
solset = h5.getSolset('sol000')
if soltab == 'all':
soltabnames = solset.getSoltabNames()
else:
soltabnames = [soltab]
for soltabname in soltabnames:
soltab = solset.getSoltab(soltabname)
soltab_swap_freq_time(soltab)
# Authors: # Francesco de Gasperin _author = "Francesco de Gasperin ([email protected])" import sys, os, time import numpy as np import logging import losoto._version import losoto._logging from losoto.h5parm import h5parm if os.path.isfile('test.h5'): os.system('rm test.h5') # general h5parm library logging.info("Create a new H5parm") H5 = h5parm('test.h5', readonly=False) logging.info("Close H5parm") H5.close() logging.info("Open in read-only mode") H5 = h5parm('test.h5', readonly=True) H5.close() # solsets print("###########################################") logging.info('### Solset') H5 = h5parm('test.h5', readonly=False) logging.info("Create solset") H5.makeSolset('ssTest') logging.info("Create solsets (using same name)") H5.makeSolset('ssTest') logging.info("Create solsets (using default name)")
# Check options
if len(args) != 2:
opt.print_help()
sys.exit()
if options.verbose: _logging.setLevel("debug")
# Check input H5parm file
h5parmFile = args[0]
if not os.path.exists(h5parmFile):
logging.critical('Input H5parm file not found.')
sys.exit(1)
logging.info("Input H5parm filename = "+h5parmFile)
# Open the h5parm file and get solution set names
h5parm_in = h5parm(h5parmFile, readonly = True)
solsetNames = h5parm_in.getSolsetNames()
# Check input parmdb file
globaldbFile = args[1]
if not os.path.exists(globaldbFile):
logging.critical('Input globaldb/SB file not found.')
sys.exit(1)
logging.info("Input globaldb/SB filename = "+globaldbFile)
# Check input solution set name
solsetName = options.solset
if solsetName not in solsetNames:
logging.critical('The solution set "'+solsetName+'" was not found in input H5parm file.')
sys.exit(1)
logging.info("Solution set name = "+solsetName)
import optparse
opt = optparse.OptionParser(usage='%prog -p H5parm [-s solset] -g parmdb [-n 100]\n'\
+_author, version='%prog '+_version.__version__)
opt.add_option('-p', '--h5parm', help='H5parm name', type='string', default='')
opt.add_option('-g', '--parmdb', help='Parmdb name', type='string', default='')
opt.add_option('-s', '--solset', help='Solution-set name (default=sol000)', type='string', default='sol000')
opt.add_option('-n', '--numiter', help='Number of iterations (default=100)', type=int, default=100)
(options, args) = opt.parse_args()
_logging.setLevel('debug')
n = options.numiter
solset = options.solset
h5parmFile = options.h5parm
H5 = h5parm(h5parmFile, readonly=False)
H = solFetcher(H5.getSoltab(solset,'rotation000'))
logging.info("H5parm filename = "+h5parmFile)
parmdbFile = options.parmdb
P = lofar.parmdb.parmdb(parmdbFile)
P2 = lofar.parmdb.parmdb('tmp.parmdb', create=True)
logging.info("parmdb filename = "+parmdbFile)
######################################################
logging.info("### Read all frequencies for a pol/dir/station")
start = time.clock()
for i in xrange(n):
Pfreqs = P.getValuesGrid('RotationAngle:CS001LBA:3C196')['RotationAngle:CS001LBA:3C196']['freqs']
elapsed = (time.clock() - start)
def __init__(self, h5file, logging_instance, parent=None):
super(H5PlotGUI, self).__init__(parent)
self.logger = logging_instance
self.figures = []
self.h5parm = lh5.h5parm(h5file)
self.solset_labels = self.h5parm.getSolsetNames()
self.solset = self.h5parm.getSolset('sol000')
self.soltab_labels = self.solset.getSoltabNames()
self.soltab = self.solset.getSoltab(self.soltab_labels[0])
self.stations = self.soltab.getValues()[1]['ant']
self.refant = 'CS001HBA0'
self.wrapphase = True
self.stcache = SoltabCache(self.soltab.getValues(), self.soltab.getAxesNames())
rvals, raxes = reorder_soltab(self.soltab)
self.stcache.update(rvals, raxes)
self.move(300, 300)
self.setWindowTitle('H5Plot')
self.solset_label = QLabel('SolSet: ')
self.solset_picker = QComboBox()
for l in self.solset_labels:
self.solset_picker.addItem(l)
self.solset_picker.activated.connect(self._solset_picker_event)
self.soltab_label_y = QLabel('Plot ')
self.soltab_label_x = QLabel(' vs ')
self.soltab_picker = QComboBox()
for l in self.soltab_labels:
self.soltab_picker.addItem(l)
self.soltab_picker.activated.connect(self._soltab_picker_event)
self.axis_picker = QComboBox()
self.axis_picker.addItems(['time', 'freq'])
self.axis_picker.activated.connect(self._axis_picker_event)
self.axis = 'time'
self.refant_label = QLabel('Ref. Ant. ')
self.refant_picker = QComboBox()
self.refant_picker.addItems(self.stations)
self.refant_picker.activated.connect(self._refant_picker_event)
self.phasewrap_box = QCheckBox('Wrap Phases')
self.phasewrap_box.setChecked(True)
self.phasewrap_box.setEnabled(False)
self.phasewrap_box.stateChanged.connect(self._phasewrap_event)
self.plot_button = QPushButton('Plot')
self.plot_button.clicked.connect(self._plot_button_event)
self.station_picker = QListWidget()
self.station_picker.addItems(self.stations)
self.station_picker.setCurrentRow(0)
plot_layout = QGridLayout()
plot_layout.addWidget(self.soltab_label_y, 0, 0)
plot_layout.addWidget(self.soltab_picker, 0, 1)
plot_layout.addWidget(self.soltab_label_x, 0, 2)
plot_layout.addWidget(self.axis_picker, 0, 3)
plot_layout.addWidget(self.refant_label, 1, 0)
plot_layout.addWidget(self.refant_picker, 1, 1)
plot_layout.addWidget(self.phasewrap_box, 1, 3)
layout = QFormLayout(self)
layout.addRow(self.solset_label, self.solset_picker)
layout.addRow(plot_layout)
layout.addRow(self.plot_button)
layout.addRow(self.station_picker)
def main(h5parmfile,
MSfiles,
solset_in='sol000',
solset_out='sol001',
soltab_list='phase000,amplitude000',
superstation='ST001',
restrictToCS=True):
''' Copy the gains from the phased up core back to all core stations.
Args:
h5parmfile (str): input H5Parm
MSfiles (list): list of (a) measurement set(s) from which the stations are read.
solset_in (str): input solset to process.
solset_out (str): output solset with the core stations added.
soltab_list (list): list of strings, containing the soltabs to be processed.
superstation (str): name of the phased up station.
restrictToCS (bool): only do this operation for stations starting with CS. Default: True
Returns:
0 or 1 (int): 0 if succesfull, 1 if an error occured.
'''
mslist = MSfiles.lstrip('[').rstrip(']').replace(' ',
'').replace("'",
"").split(',')
soltab_list = soltab_list.split(',')
if len(mslist) == 0:
logging.error("Did not find any existing directory in input MS list!")
return 1
else:
MSfile = mslist[0]
if not os.path.exists(h5parmfile):
logging.error("H5parm file %s doesn't exist!" % h5parmfile)
return 1
if not os.path.exists(MSfile):
logging.error("MS file %s doesn't exist!" % MSfile)
return 1
if solset_in == solset_out:
logging.error("Output solset has to be different from input solset!")
return 1
# Open up the h5parm, get an example value
data = h5parm(h5parmfile, readonly=False)
# Create a new solset for the data
if solset_out not in data.getSolsetNames():
new_station_names = makesolset(MSfile, data, solset_out)
# loading solset
solset = data.getSolset(solset_in)
OutSolset = data.getSolset(solset_out)
station_names = solset.getAnt().keys()
if superstation not in station_names:
logging.error("Couldn't find station " + str(superstation))
return 1
# start copying
for soltab_name in soltab_list:
logging.info("Running copySTgains_toCS on: " + soltab_name)
soltab = solset.getSoltab(soltab_name)
STindex = soltab.getAxisValues(
'ant', ignoreSelection=True).tolist().index(superstation)
if 'clock' in soltab_name or 'tec' in soltab_name:
for vals, weights, coord, selection in soltab.getValuesIter(
returnAxes=['time', 'ant'], weight=True):
vals = reorderAxes(vals, soltab.getAxesNames(),
['time', 'ant'])
weights = reorderAxes(weights, soltab.getAxesNames(),
['time', 'ant'])
elif 'amplitude' in soltab_name or 'phase' in soltab_name:
if 'pol' in soltab.getAxesNames():
for vals, weights, coord, selection in soltab.getValuesIter(
returnAxes=['pol', 'ant', 'freq',
'time'], weight=True):
vals = reorderAxes(vals, soltab.getAxesNames(),
['time', 'ant', 'freq', 'pol'])
weights = reorderAxes(weights, soltab.getAxesNames(),
['time', 'ant', 'freq', 'pol'])
else: # in case we have have no polarization axis, so scalarphase-type
for vals, weights, coord, selection in soltab.getValuesIter(
returnAxes=['ant', 'freq', 'time'], weight=True):
vals = reorderAxes(vals, soltab.getAxesNames(),
['time', 'ant', 'freq'])
weights = reorderAxes(weights, soltab.getAxesNames(),
['time', 'ant', 'freq'])
else:
logging.error(
'No phase or amplitude soltab has been found or specified.')
return 1
dimension = np.shape(vals)
if 'clock' in soltab_name or 'tec' in soltab_name:
new_vals = np.ndarray(shape=(dimension[0], len(new_station_names)))
new_weights = np.ndarray(shape=(dimension[0],
len(new_station_names)))
if 'amplitude' in soltab_name or 'phase' in soltab_name:
if 'pol' in soltab.getAxesNames():
new_vals = np.ndarray(shape=(dimension[0],
len(new_station_names),
dimension[2], dimension[3]))
new_weights = np.ndarray(shape=(dimension[0],
len(new_station_names),
dimension[2], dimension[3]))
else:
new_vals = np.ndarray(shape=(dimension[0],
len(new_station_names),
dimension[2]))
new_weights = np.ndarray(shape=(dimension[0],
len(new_station_names),
dimension[2]))
for i, new_station in enumerate(new_station_names):
if new_station in station_names:
ant_index = soltab.getAxisValues(
'ant', ignoreSelection=True).tolist().index(new_station)
if 'clock' in soltab_name or 'tec' in soltab_name:
new_vals[:, i] = vals[:, ant_index]
new_weights[:, i] = weights[:, ant_index]
if 'amplitude' in soltab_name or 'phase' in soltab_name:
if 'pol' in soltab.getAxesNames():
new_vals[:, i, :, :] = vals[:, ant_index, :, :]
new_weights[:, i, :, :] = weights[:, ant_index, :, :]
else:
new_vals[:, i, :] = vals[:, ant_index, :]
new_weights[:, i, :] = weights[:, ant_index, :]
else:
if restrictToCS and 'CS' not in new_station:
logging.info('RestrictToCS: Omitting station ' +
new_station)
continue
logging.info('Adding ' + str(soltab_name) + ' to ' +
new_station)
if 'clock' in soltab_name or 'tec' in soltab_name:
new_vals[:, i] = vals[:, STindex]
new_weights[:, i] = weights[:, STindex]
if 'amplitude' in soltab_name or 'phase' in soltab_name:
if 'pol' in soltab.getAxesNames():
new_vals[:, i, :, :] = vals[:, STindex, :, :]
new_weights[:, i, :, :] = weights[:, STindex, :, :]
else:
new_vals[:, i, :] = vals[:, STindex, :]
new_weights[:, i, :] = weights[:, STindex, :]
if 'clock' in soltab_name:
new_soltab = OutSolset.makeSoltab(
soltype='clock',
soltabName=soltab_name,
axesNames=['time', 'ant'],
axesVals=[soltab.time, new_station_names],
vals=new_vals,
weights=new_weights)
pass
elif 'tec' in soltab_name:
new_soltab = OutSolset.makeSoltab(
soltype='tec',
soltabName=soltab_name,
axesNames=['time', 'ant'],
axesVals=[soltab.time, new_station_names],
vals=new_vals,
weights=new_weights)
elif 'amplitude' in soltab_name:
if 'pol' in soltab.getAxesNames():
new_soltab = OutSolset.makeSoltab(
soltype='amplitude',
soltabName=soltab_name,
axesNames=['time', 'ant', 'freq', 'pol'],
axesVals=[
soltab.time, new_station_names, soltab.freq, soltab.pol
],
vals=new_vals,
weights=new_weights)
else:
new_soltab = OutSolset.makeSoltab(
soltype='amplitude',
soltabName=soltab_name,
axesNames=['time', 'ant', 'freq'],
axesVals=[soltab.time, new_station_names, soltab.freq],
vals=new_vals,
weights=new_weights)
elif 'phase' in soltab_name:
if 'pol' in soltab.getAxesNames():
new_soltab = OutSolset.makeSoltab(
soltype='phase',
soltabName=soltab_name,
axesNames=['time', 'ant', 'freq', 'pol'],
axesVals=[
soltab.time, new_station_names, soltab.freq, soltab.pol
],
vals=new_vals,
weights=new_weights)
else:
new_soltab = OutSolset.makeSoltab(
soltype='phase',
soltabName=soltab_name,
axesNames=['time', 'ant', 'freq'],
axesVals=[soltab.time, new_station_names, soltab.freq],
vals=new_vals,
weights=new_weights)
soltab = 0
return 0
fieldFile = os.path.join(globaldbFile,'FIELD')
if not os.path.isdir(fieldFile):
logging.critical('Missing FIELD table.')
sys.exit(1)
skydbFile = os.path.join(globaldbFile,'sky')
if not os.path.isdir(skydbFile):
logging.critical('Missing skydb table.')
sys.exit(1)
# Make a list of all available instrument tables (only 1 for a standard MS)
instrumentdbFiles = [ instrumentdbFile for instrumentdbFile in \
glob.glob(os.path.join(globaldbFile,options.instrument)) \
if os.path.isdir(instrumentdbFile) ]
# open/create the h5parm file and the solution-set
h5parm = h5parm(h5parmFile, readonly = False, complevel = complevel)
solsetName = options.solset
solset = h5parm.makeSolset(solsetName)
# Create tables using the first instrumentdb
# TODO: all the instrument tables should be checked
pdb = lofar.parmdb.parmdb(instrumentdbFiles[0])
solTypes = list(set(x[0] for x in (x.split(":") for x in pdb.getNames())))
logging.info('Found solution types: '+', '.join(solTypes))
# rewrite solTypes in order to put together
# Gain <-> DirectionalGain
# CommonRotationAngle <-> RotationAngle
# CommonScalarPhase <-> ScalarPhase
#chi = numpy.sum(angle)
residual = numpy.mod(phase-phase_total+pi,2.*pi)-pi
#print residual
chi = numpy.sum(numpy.abs(residual))/len(residual)
return [fitresult[0], fitresult[1],fitresult[2], chi]
#import lofar.expion.fitting as fitting
pi = numpy.pi
c = 2.99792458e8
ionmodel = h5parm('scalarphase.h',readonly=True)
solsetNames = ionmodel.getSolsets()
for solsetName in solsetNames:
print solsetName
solset = ionmodel.getSolset(solsetName)
soltabs = ionmodel.getSoltabs('sol000')
scalarphasetab = ionmodel.getSoltab('sol000','scalarphase000')
anttab = ionmodel.getAnt('sol000')
print scalarphasetab.freq[:]
#print scalarphasetab.time[:]
#print scalarphasetab.val[:]
#print scalarphasetab.ant[:]
def main(instrument_name, normalize=True):
if type(normalize) is str:
if normalize.lower() == 'true':
normalize = True
else:
normalize = False
H = h5parm(instrument_name, readonly=False)
solset = H.getSolset('sol000')
soltab = solset.getSoltab('amplitude000')
parms = soltab.val[:]
# Check for NaNs and zeros. If found, set to 1
ntimes = len(soltab.time[:])
nfreqs = len(soltab.freq[:])
initial_flagged_indx = numpy.where(numpy.logical_or(numpy.isnan(parms), parms == 0.0))
initial_unflagged_indx = numpy.where(numpy.logical_and(~numpy.isnan(parms), parms != 0.0))
parms[initial_flagged_indx] = 1.0
# Get station names
antenna_list = soltab.ant[:]
axis_names = soltab.getAxesNames()
for pol in range(len(soltab.pol[:])):
for istat, antenna in enumerate(antenna_list):
soltab.setSelection(ant=[antenna])
channel_amp_orig = soltab.val[:] # ['time', 'freq', 'ant', 'dir', 'pol']
# Find flagged solutions and set to 1.0
channel_amp_interp = []
for chan in range(nfreqs):
unflagged_times = numpy.where(numpy.logical_and(channel_amp_orig[:, chan, 0, 0, pol] > 0.0,
soltab.weight[:, chan, 0, 0, pol] != 0.0))
flagged_times = numpy.where(numpy.logical_or(numpy.logical_or(
numpy.isnan(channel_amp_orig[:, chan, 0, 0, pol]),
channel_amp_orig[:, chan, 0, 0, pol] == 0.0),
soltab.weight[:, chan, 0, 0, pol] == 0.0))
if numpy.any(flagged_times):
channel_amp_orig[flagged_times, chan, 0, 0, pol] = 1.0
if numpy.any(unflagged_times):
channel_amp_interp.append(channel_amp_orig[:, chan, 0, 0, pol])
else:
channel_amp_interp.append(None)
# now find the bad data
if ntimes > 5:
pool = multiprocessing.Pool()
results = pool.map(spline1D, channel_amp_interp)
pool.close()
pool.join()
for chan, (amp_cleaned, model, noisevec, scatter, n_knots, idxbad, weights) in enumerate(results):
# put back the results
if amp_cleaned is None:
amp_cleaned = channel_amp_orig[:, chan, 0, 0, pol]
parms[:, chan, istat, 0, pol] = numpy.copy(amp_cleaned)
if nfreqs > 5: # Do 2D smooth
channel_amp_orig = parms[:, :, istat, 0, pol]
# Smooth
channel_amp_interp = []
unflagged_sols = numpy.where(channel_amp_orig != 1.0)
if numpy.any(unflagged_sols):
# Set flagged solutions to 1.0
flagged_sols = numpy.where(numpy.logical_or(channel_amp_orig == 1.0, channel_amp_orig <= 0.0))
channel_amp_orig[flagged_sols] = 1.0
# Filter
amp_cleaned, amp_median, baddata = median2Dampfilter(channel_amp_orig.transpose([1, 0]))
amp_cleaned = amp_cleaned.transpose([1, 0])
amp_cleaned[flagged_sols] = 1.0
parms[:, :, istat, 0, pol] = numpy.copy(amp_cleaned)
# Normalize the amplitude solutions to a mean of one across all channels
if normalize:
# First find the normalization factor from unflagged solutions
amplist = []
norm_factor = 1.0/(numpy.nanmean(parms[initial_unflagged_indx]))
print "smooth_amps_spline.py: Normalization-Factor is:", norm_factor
parms *= norm_factor
# Clip extremely low amplitude solutions to prevent very high
# amplitudes in the corrected data
unflagged = numpy.where(~numpy.isnan(parms))
low_ind = numpy.where(parms[unflagged] < 0.2)
parms[unflagged][low_ind] = 0.2
# Make sure flagged solutions are still flagged
parms[initial_flagged_indx] = numpy.nan
# Write the results to the output soltab
try:
new_soltab = solset.getSoltab('amplitude001')
new_soltab.delete()
except:
pass
st = solset.makeSoltab('amplitude', 'amplitude001', axesNames=axis_names,
axesVals=[soltab.time[:], soltab.freq[:], antenna_list,
soltab.dir[:], soltab.pol[:]], vals=parms,
weights=numpy.ones(parms.shape))
reordered = reorderAxes(st.getValues()[0], order_old, order_new)
reordered2 = {}
for k in order_new:
reordered2[k] = st.axes[k]
st.axes = reordered2
st.axesNames = order_new
st_new = (reordered, st.getValues()[1])
return st_new, order_new
def wrap_phase(phase):
wphase = (phase + np.pi) % (2 * np.pi) - np.pi
return wphase
if __name__ == '__main__':
FILENAME = sys.argv[1]
H5FILE = lh5.h5parm(FILENAME, readonly=True)
# Set up for logging output.
LOGGER = logging.getLogger('H5plot_logger')
#LOGGER.setLevel(logging.INFO)
LOGGER.setLevel(logging.DEBUG)
formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
LOGFILEH = logging.FileHandler('h5plot.log')
LOGFILEH.setLevel(logging.DEBUG)
LOGFILEH.setFormatter(formatter)
LOGGER.addHandler(LOGFILEH)
LOGGER.info('Successfully opened %s', FILENAME)
SOLSETS = H5FILE.getSolsetNames()
print(('Found solset(s) '+ ', '.join(SOLSETS)))
# Mirror at left edge.
idx = min(ndata-1, half_window-i)
sol[i] = ampl[idx]
# Mirror at right edge
idx = max(0, ndata-2-i)
sol[ndata+half_window+i] = ampl[idx]
for i in range(len(ampl)):
#print i, i+half_window
std[i] = numpy.median(sol[i:i+(2*half_window)])
return std
ionmodel = h5parm(globaldbname ,readonly=True)
amptab = ionmodel.getSoltab('sol000','amplitude000')
# save for quicker access
#numpy.save('amplitude_array.npy',ionmodel.amplitudes)
#amplitude_arraytmp = numpy.load('amplitude_array.npy')
amplitude_arraytmp = amptab.val[:]
print numpy.shape(amplitude_arraytmp)
print len(amptab.ant[:]), len(amptab.freq[:]) , len(amptab.time[:])
# in reinout version 62 is the number of freqs, 75 is time, 2912/8 is averaging in freq
#amplitude_array = rebin(amplitude_arraytmp, (75, 2912/8, 62, 1, 2))
amplitude_array = rebin(amplitude_arraytmp, (2, 1, len(amptab.ant[:]), len(amptab.freq[:])/n_chan , len(amptab.time[:])))
def run(obs,
method,
h5parmFilename,
maxdtec=0.5,
maxvtec=50,
alphaIon=11 / 3,
hIon=250e3,
vIon=20,
seed=0,
fitsFilename=None,
stepname='tec',
angRes=60,
expfolder='',
ncpu=0):
"""
Simulate TEC values and store them to a h5parm.
Parameters
----------
method : str
Method to use:
"fits": read TEC values from the FITS cube specified by fitsFilename
"tid": generate a traveling ionospheric disturbance (TID) wave
"turbulence": generate a turbulent ionosphere
h5parmFilename : str
Filename of output h5parm file.
maxdtec : float, optional. Default = 0.5
Maximum screen dTEC per timestep in TECU.
maxvtec: float, optional. Default = 50.
Highest vTEC in daily modulation in TECU.
alphaIon: float, optional. Default = 11/3.
Ionosphere power spectrum exponent. Is 11/3 for Kolmogorov,
de Gasperin and Mevius found 3.89 for LOFAR.
hIon : float, optional. Default = 250 km
Height of thin layer ionoshpere.
vIono : float, optional. Default = 20 m/s
Velocity of tecscreen. This controls the tec variation frequency.
seed: int, optional.
Random screen seed. Use for reproducibility.
fitsFilename : str, optional
Filename of input FITS cube with dTEC solutions.
stepname _ str, optional
Name of step to use in DPPP parset
angRes : float, optional. Default = 60.
Angular resolution of the screen [arcsec]. Only for turbulent model.
expfolder : str, optional. Default = None
Export the tecscreen data to this folder for plotting. Depending on
system memory, this will not work for very large/highres screens.
Only for 'turbulence' method.
ncpu : int, optional
Number of cores to use, by default all available.
"""
# TODO : Test TID and polynomial method for multi-ms usage
method = method.lower()
if ncpu == 0:
ncpu = mp.cpu_count()
# Get sky model properties
ras, decs = obs.get_patch_coords()
source_names = obs.get_patch_names()
ants = obs.stations
sp = obs.stationpositions
times = obs.get_times()
tecvals = np.zeros((len(times), len(ants), len(ras)))
weights = np.ones_like(tecvals)
if method == 'turbulence':
directions = np.array([ras, decs]).T
tecvals = comoving_tecscreen(sp,
directions,
times,
alpha=alphaIon,
angRes=angRes,
hIon=hIon,
vIon=vIon,
maxvtec=maxvtec,
maxdtec=maxdtec,
ncpu=ncpu,
expfolder=expfolder,
seed=seed)
elif method == 'fits':
# Load solutions from FITS cube
hdu = pyfits.open(fitsFilename, memmap=False)
data = hdu[0].data
header = hdu[0].header
w = wcs.WCS(header)
ntimes, _, nstations, ny, nx = data.shape
# Check that number of stations in input FITS cube matches MS
if nstations != len(ants):
logger.error('Number of stations in input FITS cube does not '
'match that in the input MS')
return 1
# Get solutions at the source coords
for d, (ra_deg, dec_deg) in enumerate(zip(ras, decs)):
ra_dec = np.array([[ra_deg, dec_deg, 0, 0, 0]])
x = int(w.wcs_world2pix(ra_dec, 0)[0][0])
y = int(w.wcs_world2pix(ra_dec, 0)[0][1])
if x < 0 or x > nx or y < 0 or y > ny:
tecvals[:, :, d, :] = 0.0
weights[:, :, d, :] = 0.0
continue
for t in range(ntimes):
for s in range(nstations):
tecvals[t, s, d] = data[t, 0, s, y, x]
tecvals = daytime_tec_modulation(times)[:, np.newaxis, np.newaxis] * (
tecvals + maxvtec)
elif method == 'tid':
# Properties of TID wave
tidLen = 200e3
tidVel = 500e3 / 3600,
tid_prop = [maxdtec, tidLen, tidVel]
# Generate solutions for TID wave
A12 = EarthLocation(lat=52.91 * u.deg,
lon=6.87 * u.deg,
height=1 * u.m)
mjd = Time(times / (3600.0 * 24.0), format="mjd")
aa = AltAz(location=A12, obstime=mjd)
altazcoord = []
pool = mp.Pool(processes=ncpu)
radec = [(r, d, aa) for r, d in zip(ras, decs)]
altazcoord = pool.map(_getaltaz, radec)
gettec_args = [(a, sp, A12, times, *tid_prop) for a in altazcoord]
alltec = pool.map(_gettec, gettec_args)
pool.close()
pool.join()
alltec = np.array(alltec)
# Fill the axis arrays
tecvals = alltec[:, :, 0, :].transpose([2, 1, 0]) # [:,:,:,0]
# convert to vTEC
tecvals = daytime_tec_modulation(times)[:, np.newaxis, np.newaxis] * (
tecvals + maxvtec)
else:
logger.error('method "{}" not understood'.format(method))
return 1
# Write tec values to h5parm file as DPPP input
ho = h5parm(h5parmFilename, readonly=False)
if 'sol000' in ho.getSolsetNames():
solset = ho.getSolset('sol000')
else:
solset = ho.makeSolset(solsetName='sol000')
if 'tec000' in solset.getSoltabNames():
logger.info('''Solution-table tec000 is already present in
{}. It will be overwritten.'''.format(h5parmFilename +
'/sol000'))
solset.getSoltab('tec000').delete()
st = solset.makeSoltab('tec',
'tec000',
axesNames=['time', 'ant', 'dir'],
axesVals=[times, ants, source_names],
vals=tecvals,
weights=weights)
antennaTable = solset.obj._f_get_child('antenna')
antennaTable.append(list(zip(*(ants, sp))))
sourceTable = solset.obj._f_get_child('source')
vals = [[ra, dec] for ra, dec in zip(ras, decs)]
sourceTable.append(list(zip(*(source_names, vals))))
# Add CREATE entry to history
soltabs = solset.getSoltabs()
for st in soltabs:
st.addHistory('CREATE (by TEC operation of LoSiTo from obs {0} '
'and method="{{1}}")'.format(h5parmFilename, method))
ho.close()
# Update predict parset parameters for the obs
obs.add_to_parset(stepname, 'tec000', h5parmFilename)
return 0
args.h5parmFiles = args.h5parmFiles[0].strip('[]').split(',')
args.h5parmFiles = [f.strip() for f in args.h5parmFiles]
# open input
all_soltabs = {} # {'amplitude':[soltab1, soltab2], 'phase':[soltab3]}
all_axes_names = {
} # {'amplitude':['dir','time','ant','freq'], 'phase':['dir','time','freq']} # dir is the first by construction
all_axes_vals = {
} # {'time':[[1,2,34,5],[1,3,5]],'freq':[[1,2,3,4,5],[1,3,5]]}
pointingNames = []
antennaNames = []
pointingDirections = []
antennaPositions = []
for h5parmFile in args.h5parmFiles:
logging.info('Working on %s...' % h5parmFile)
h5 = h5parm(h5parmFile.replace("'", ""), readonly=True)
solset = h5.getSolset(args.insolset)
insoltabs = solset.getSoltabNames()
# restrict to requested insoltabs
if args.insoltab is not None:
insoltabs = [
insoltab for insoltab in insoltabs if insoltab in args.insoltab
]
if len(insoltabs) == 0:
logging.critical('No soltabs found.')
sys.exit()
for insoltab in insoltabs:
soltab = solset.getSoltab(insoltab)
# classify soltabs
typ = soltab.getType()
sys.exit(1)
skydbFile = os.path.join(inMSs[0], 'sky')
if not os.path.isdir(skydbFile):
logging.warning(
'No sky table found. (Direction-dependent parameters will not work.)'
)
skydbFile = None
#generate list of parmDB-filenames
parmDBnames = [MS.rstrip('/') + instrument for MS in inMSs]
#create and fill the hdf5-file:
solset = parmDBs2h5parm(hdf5File,
parmDBnames,
antennaFile,
fieldFile,
skydbFile,
compression=compression,
solsetName=solsetName)
# Add CREATE entry to history
h5parmDB = h5parm(hdf5File, readonly=False)
soltabs = h5parmDB.getSoltabs(solset=solset)
for st in soltabs:
sw = solWriter(soltabs[st])
sw.addHistory(
'CREATE (by PipelineStep_losotoImporter from %s / %s - %s)' %
(os.path.abspath(''), os.path.basename(
parmDBnames[0]), os.path.basename(parmDBnames[-1])))
h5parmDB.close()
parser.add_argument('--outsolset', '-o', default=None, dest='outsolset', help='Output solset name [default: same as insolset]')
parser.add_argument('--root', '-r', default='output', dest='root', help='Root of output h5parm names; e.g., root_0.h5, root_1.h5, etc. [default: output]')
parser.add_argument('--verbose', '-V', '-v', default=False, action='store_true', help='Go Vebose! (default=False)')
parser.add_argument('--clobber', '-c', default=False, action='store_true', help='Replace exising outh5parm files (default=False)')
args = parser.parse_args()
if len(args.h5parmFile) < 1:
parser.print_help()
sys.exit(1)
if args.verbose: _logging.setLevel("debug")
################################
# read input table
h5 = h5parm(args.h5parmFile, readonly=True)
solset = h5.getSolset(args.insolset)
insoltabs = solset.getSoltabNames()
soltabs = []
for insoltab in insoltabs:
soltab = solset.getSoltab(insoltab)
soltabs.append( soltab )
pointingNames = []; antennaNames = []
pointingDirections = []; antennaPositions = []
ants = solset.getAnt()
sous = solset.getSou()
for k,v in list(sous.items()):
if k not in pointingNames:
pointingNames.append(k)
pointingDirections.append(v)
for k, v in list(ants.items()):
# Check is all the necessary files are available
antennaFile = os.path.join(inMSs[0],'ANTENNA')
if not os.path.isdir(antennaFile):
logging.critical('Missing ANTENNA table.')
sys.exit(1)
fieldFile = os.path.join(inMSs[0],'FIELD')
if not os.path.isdir(fieldFile):
logging.critical('Missing FIELD table.')
sys.exit(1)
skydbFile = os.path.join(inMSs[0],'sky')
if not os.path.isdir(skydbFile):
logging.critical('Missing sky table.')
sys.exit(1)
#generate list of parmDB-filenames
parmDBnames = [ MS.rstrip('/')+instrument for MS in inMSs ]
#create and fill the hdf5-file:
solset = parmDBs2h5parm(hdf5File, parmDBnames, antennaFile, fieldFile, skydbFile, compression=compression, solsetName=solsetName)
# Add CREATE entry to history
h5parmDB = h5parm(hdf5File, readonly = False)
soltabs = h5parmDB.getSoltabs(solset=solset)
for st in soltabs:
sw = solWriter(soltabs[st])
sw.addHistory('CREATE (by PipelineStep_losotoImporter from %s / %s - %s)' % (os.path.abspath(''),
os.path.basename(parmDBnames[0]), os.path.basename(parmDBnames[-1]) ) )
h5parmDB.close()
def parmDBs2h5parm(h5parmName,
parmDBs,
antennaFile,
fieldFile,
skydbFile=None,
compression=5,
solsetName=None):
"""
Write the contents of a list of parmDBs into a losoto-style hdf5-file
h5parmName - name (path) of the hdf5 file to generate
parmDBs - list with the names of the parmDBs
antennaFile - name (path) of an ANTENNA table of the observation
fieldFile - name (path) of a FIELD table of the observation
skydbFile - name (path) of a skydb table of the calibration run (Needed for direction dependent parameters)
compresion - compression level for the hdf5-file (0 - none ; 9 - highest)
solsetName - name of the solset to generate (default: "sol000")
"""
# open/create the h5parm file and the solution-set
h5parmDB = h5parm(h5parmName, readonly=False, complevel=compression)
solset = h5parmDB.makeSolset(solsetName)
#open the first instrument table, so that we know where to look for names and stuff
firstInst = pdb.parmdb(parmDBs[0])
# get unique list of solution types
solTypes = list(set(x1.split(":")[0] for x1 in firstInst.getNames()))
# rewrite solTypes in order to put together
# Gain <-> DirectionalGain
# CommonRotationAngle <-> RotationAngle
# CommonScalarPhase <-> ScalarPhase
# it also separate Real/Imag/Ampl/Phase into different solTypes
if "Gain" in solTypes:
solTypes.remove('Gain')
solTypes.append('*Gain:*:Real')
solTypes.append('*Gain:*:Imag')
solTypes.append('*Gain:*:Ampl')
solTypes.append('*Gain:*:Phase')
if "DirectionalGain" in solTypes:
solTypes.remove('DirectionalGain')
solTypes.append('*Gain:*:Real')
solTypes.append('*Gain:*:Imag')
solTypes.append('*Gain:*:Ampl')
solTypes.append('*Gain:*:Phase')
if "RotationAngle" in solTypes:
solTypes.remove('RotationAngle')
solTypes.append('*RotationAngle')
if "CommonRotationAngle" in solTypes:
solTypes.remove('CommonRotationAngle')
solTypes.append('*RotationAngle')
if "RotationMeasure" in solTypes:
solTypes.remove('RotationMeasure')
solTypes.append('*RotationMeasure')
if "ScalarPhase" in solTypes:
solTypes.remove('ScalarPhase')
solTypes.append('*ScalarPhase')
if "CommonScalarPhase" in solTypes:
solTypes.remove('CommonScalarPhase')
solTypes.append('*ScalarPhase')
if "CommonScalarAmplitude" in solTypes:
solTypes.remove('CommonScalarAmplitude')
solTypes.append('*ScalarAmplitude')
# and remove duplicate entries
solTypes = list(set(solTypes))
for solType in solTypes:
if len(firstInst.getNames(solType + ':*')) == 0: continue
pols = set()
dirs = set()
ants = set()
freqs = set()
times = set()
ptype = set()
for pDBname in parmDBs:
instrumentdb = pdb.parmdb(pDBname)
# create the axes grid, necessary if not all entries have the same axes lenght
data = instrumentdb.getValuesGrid(solType + ':*')
for solEntry in data:
pol, dir, ant, parm = parmdbToAxes(solEntry)
if pol != None: pols |= set([pol])
if dir != None: dirs |= set([dir])
if ant != None: ants |= set([ant])
freqs |= set(data[solEntry]['freqs'])
times |= set(data[solEntry]['times'])
#close the parmDB
instrumentdb = 0
pols = np.sort(list(pols))
dirs = np.sort(list(dirs))
ants = np.sort(list(ants))
freqs = np.sort(list(freqs))
times = np.sort(list(times))
shape = [
i
for i in (len(pols), len(dirs), len(ants), len(freqs), len(times))
if i != 0
]
vals = np.empty(shape)
vals[:] = np.nan
weights = np.zeros(shape)
for pDBname in parmDBs:
instrumentdb = pdb.parmdb(pDBname)
# fill the values
data = instrumentdb.getValuesGrid(solType + ':*')
if 'Real' in solType:
dataIm = instrumentdb.getValuesGrid(
solType.replace('Real', 'Imag') + ':*')
if 'Imag' in solType:
dataRe = instrumentdb.getValuesGrid(
solType.replace('Imag', 'Real') + ':*')
for solEntry in data:
pol, dir, ant, parm = parmdbToAxes(solEntry)
ptype |= set([solEntry.split(':')[0]
]) # original parmdb solution type
freq = data[solEntry]['freqs']
time = data[solEntry]['times']
val = data[solEntry]['values']
# convert Real and Imag in Amp and Phase respectively
if parm == 'Real':
solEntryIm = solEntry.replace('Real', 'Imag')
valI = dataIm[solEntryIm]['values']
val = np.sqrt((val**2) + (valI**2))
if parm == 'Imag':
solEntryRe = solEntry.replace('Imag', 'Real')
valR = dataRe[solEntryRe]['values']
val = np.arctan2(val, valR)
coords = []
if pol != None:
polCoord = np.searchsorted(pols, pol)
coords.append(polCoord)
if dir != None:
dirCoord = np.searchsorted(dirs, dir)
coords.append(dirCoord)
if ant != None:
antCoord = np.searchsorted(ants, ant)
coords.append(antCoord)
freqCoord = np.searchsorted(freqs, freq)
timeCoord = np.searchsorted(times, time)
vals[tuple(coords)][np.ix_(freqCoord, timeCoord)] = val.T
weights[tuple(coords)][np.ix_(freqCoord, timeCoord)] = 1
#close the parmDB
instrumentdb = 0
vals = np.nan_to_num(vals) # replace nans with 0 (flagged later)
if solType == '*RotationAngle':
np.putmask(weights, vals == 0., 0) # flag where val=0
h5parmDB.makeSoltab(solset, 'rotation', axesNames=['dir','ant','freq','time'], \
axesVals=[dirs,ants,freqs,times], vals=vals, weights=weights, parmdbType=', '.join(list(ptype)))
elif solType == '*RotationMeasure':
np.putmask(weights, vals == 0., 0) # flag where val=0
h5parm.makeSoltab(solset, 'rotationmeasure', axesNames=['dir','ant','freq','time'], \
axesVals=[dirs,ants,freqs,times], vals=vals, weights=weights, parmdbType=', '.join(list(ptype)))
elif solType == '*ScalarPhase':
np.putmask(weights, vals == 0., 0)
h5parmDB.makeSoltab(solset, 'scalarphase', axesNames=['dir','ant','freq','time'], \
axesVals=[dirs,ants,freqs,times], vals=vals, weights=weights, parmdbType=', '.join(list(ptype)))
elif solType == '*ScalarAmplitude':
np.putmask(weights, vals == 0., 0)
h5parm.makeSoltab(solset, 'scalaramplitude', axesNames=['dir','ant','freq','time'], \
axesVals=[dirs,ants,freqs,times], vals=vals, weights=weights, parmdbType=', '.join(list(ptype)))
elif solType == 'Clock':
np.putmask(weights, vals == 0., 0)
# clock may be diag or scalar
if len(pols) == 0:
h5parm.makeSoltab(solset, 'clock', axesNames=['ant','freq','time'], \
axesVals=[ants,freqs,times], vals=vals, weights=weights, parmdbType=', '.join(list(ptype)))
else:
h5parm.makeSoltab(solset, 'clock', axesNames=['pol','ant','freq','time'], \
axesVals=[pol,ants,freqs,times], vals=vals, weights=weights, parmdbType=', '.join(list(ptype)))
elif solType == 'TEC':
np.putmask(weights, vals == 0., 0)
# tec may be diag or scalar
if len(pols) == 0:
h5parm.makeSoltab(solset, 'tec', axesNames=['dir','ant','freq','time'], \
axesVals=[dirs,ants,freqs,times], vals=vals, weights=weights, parmdbType=', '.join(list(ptype)))
else:
h5parm.makeSoltab(solset, 'tec', axesNames=['pol','dir','ant','freq','time'], \
axesVals=[pols,dirs,ants,freqs,times], vals=vals, weights=weights, parmdbType=', '.join(list(ptype)))
elif solType == '*Gain:*:Real' or solType == '*Gain:*:Ampl':
np.putmask(
vals, vals == 0.,
1) # nans end up into 1s (as BBS output, flagged next line)
np.putmask(weights, vals == 1., 0) # flag where val=1
h5parmDB.makeSoltab(solset, 'amplitude', axesNames=['pol','dir','ant','freq','time'], \
axesVals=[pols,dirs,ants,freqs,times], vals=vals, weights=weights, parmdbType=', '.join(list(ptype)))
elif solType == '*Gain:*:Imag' or solType == '*Gain:*:Phase':
np.putmask(weights, vals == 0., 0) # falg where val=0
h5parmDB.makeSoltab(solset, 'phase', axesNames=['pol','dir','ant','freq','time'], \
axesVals=[pols,dirs,ants,freqs,times], vals=vals, weights=weights, parmdbType=', '.join(list(ptype)))
antennaTable = pt.table(antennaFile, ack=False)
antennaNames = antennaTable.getcol('NAME')
antennaPositions = antennaTable.getcol('POSITION')
antennaTable.close()
antennaTable = solset._f_get_child('antenna')
antennaTable.append(zip(*(antennaNames, antennaPositions)))
fieldTable = pt.table(fieldFile, ack=False)
phaseDir = fieldTable.getcol('PHASE_DIR')
pointing = phaseDir[0, 0, :]
fieldTable.close()
sourceTable = solset._f_get_child('source')
# add the field centre, that is also the direction for Gain and CommonRotationAngle
sourceTable.append([('pointing', pointing)])
dirs = []
for tab in solset._v_children:
c = solset._f_get_child(tab)
if c._v_name != 'antenna' and c._v_name != 'source':
dirs.extend(list(set(c.dir)))
# remove duplicates
dirs = list(set(dirs))
# remove any pointing (already in the table)
if 'pointing' in dirs:
dirs.remove('pointing')
if dirs != []:
if skydbFile == None:
logging.critical(
'No sky table given, but direction dependent parameters in parmDB. Exiting!'
)
sys.exit(1)
sourceFile = skydbFile + '/SOURCES'
src_table = pt.table(sourceFile, ack=False)
sub_tables = src_table.getsubtables()
vals = []
ra = dec = np.nan
has_patches_subtable = False
for sub_table in sub_tables:
if 'PATCHES' in sub_table:
has_patches_subtable = True
if has_patches_subtable:
# Read values from PATCHES subtable
src_table.close()
sourceFile = skydbFile + '/SOURCES/PATCHES'
src_table = pt.table(sourceFile, ack=False)
patch_names = src_table.getcol('PATCHNAME')
patch_ras = src_table.getcol('RA')
patch_decs = src_table.getcol('DEC')
for source in dirs:
try:
patch_indx = patch_names.index(source)
ra = patch_ras[patch_indx]
dec = patch_decs[patch_indx]
except ValueError:
ra = np.nan
dec = np.nan
vals.append([ra, dec])
src_table.close()
else:
# Try to read default values from parmdb instead
skydb = pdb.parmdb(skydbFile)
vals = []
ra = dec = np.nan
for source in dirs:
try:
ra = skydb.getDefValues('Ra:' + source)['Ra:' +
source][0][0]
dec = skydb.getDefValues('Dec:' + source)['Dec:' +
source][0][0]
except KeyError:
# Source not found in skymodel parmdb, try to find components
ra = np.array(
skydb.getDefValues('Ra:*' + source + '*').values())
dec = np.array(
skydb.getDefValues('Dec:*' + source + '*').values())
if len(ra) == 0 or len(dec) == 0:
ra = np.nan
dec = np.nan
else:
ra = ra.mean()
dec = dec.mean()
vals.append([ra, dec])
sourceTable.append(zip(*(dirs, vals)))
solsetname = solset._v_name
# close the hdf5-file
h5parmDB.close()
return solsetname
def parmDBs2h5parm(h5parmName,parmDBs,antennaFile,fieldFile,skydbFile,compression=5,solsetName=None):
"""
Write the contents of a list of parmDBs into a losoto-style hdf5-file
h5parmName - name (path) of the hdf5 file to generate
parmDBs - list with the names of the parmDBs
antennaFile - name (path) of an ANTENNA table of the observation
fieldFile - name (path) of a FIELD table of the observation
skydbFile - name (path) of a skydb table of the calibration run
compresion - compression level for the hdf5-file (0 - none ; 9 - highest)
solsetName - name of the solset to generate (default: "sol000")
"""
# open/create the h5parm file and the solution-set
h5parmDB = h5parm(h5parmName, readonly = False, complevel = compression)
solset = h5parmDB.makeSolset(solsetName)
#open the first instrument table, so that we know where to look for names and stuff
firstInst = pdb.parmdb(parmDBs[0])
# get unique list of solution types
solTypes = list(set(x1.split(":")[0] for x1 in firstInst.getNames()))
# rewrite solTypes in order to put together
# Gain <-> DirectionalGain
# CommonRotationAngle <-> RotationAngle
# CommonScalarPhase <-> ScalarPhase
# it also separate Real/Imag/Ampl/Phase into different solTypes
if "Gain" in solTypes:
solTypes.remove('Gain')
solTypes.append('*Gain:*:Real')
solTypes.append('*Gain:*:Imag')
solTypes.append('*Gain:*:Ampl')
solTypes.append('*Gain:*:Phase')
if "DirectionalGain" in solTypes:
solTypes.remove('DirectionalGain')
solTypes.append('*Gain:*:Real')
solTypes.append('*Gain:*:Imag')
solTypes.append('*Gain:*:Ampl')
solTypes.append('*Gain:*:Phase')
if "RotationAngle" in solTypes:
solTypes.remove('RotationAngle')
solTypes.append('*RotationAngle')
if "CommonRotationAngle" in solTypes:
solTypes.remove('CommonRotationAngle')
solTypes.append('*RotationAngle')
if "ScalarPhase" in solTypes:
solTypes.remove('ScalarPhase')
solTypes.append('*ScalarPhase')
if "CommonScalarPhase" in solTypes:
solTypes.remove('CommonScalarPhase')
solTypes.append('*ScalarPhase')
# and remove duplicate entries
solTypes = list(set(solTypes))
for solType in solTypes:
if len(firstInst.getNames(solType+':*')) == 0: continue
pols = set(); dirs = set(); ants = set();
freqs = set(); times = set(); ptype = set()
for pDBname in parmDBs:
instrumentdb = pdb.parmdb(pDBname)
# create the axes grid, necessary if not all entries have the same axes lenght
data = instrumentdb.getValuesGrid(solType+':*')
for solEntry in data:
pol, dir, ant, parm = parmdbToAxes(solEntry)
if pol != None: pols |= set([pol])
if dir != None: dirs |= set([dir])
if ant != None: ants |= set([ant])
freqs |= set(data[solEntry]['freqs'])
times |= set(data[solEntry]['times'])
#close the parmDB
instrumentdb = 0
pols = np.sort(list(pols)); dirs = np.sort(list(dirs));
ants = np.sort(list(ants)); freqs = np.sort(list(freqs));
times = np.sort(list(times))
shape = [i for i in (len(pols), len(dirs), len(ants), len(freqs), len(times)) if i != 0]
vals = np.empty(shape)
vals[:] = np.nan
weights = np.zeros(shape)
for pDBname in parmDBs:
instrumentdb = pdb.parmdb(pDBname)
# fill the values
data = instrumentdb.getValuesGrid(solType+':*')
if 'Real' in solType: dataIm = instrumentdb.getValuesGrid(solType.replace('Real','Imag')+':*')
if 'Imag' in solType: dataRe = instrumentdb.getValuesGrid(solType.replace('Imag','Real')+':*')
for solEntry in data:
pol, dir, ant, parm = parmdbToAxes(solEntry)
ptype |= set([solEntry.split(':')[0]]) # original parmdb solution type
freq = data[solEntry]['freqs']
time = data[solEntry]['times']
val = data[solEntry]['values']
# convert Real and Imag in Amp and Phase respectively
if parm == 'Real':
solEntryIm = solEntry.replace('Real','Imag')
valI = dataIm[solEntryIm]['values']
val = np.sqrt((val**2)+(valI**2))
if parm == 'Imag':
solEntryRe = solEntry.replace('Imag','Real')
valR = dataRe[solEntryRe]['values']
val = np.arctan2(val, valR)
coords = []
if pol != None:
polCoord = np.searchsorted(pols, pol)
coords.append(polCoord)
if dir != None:
dirCoord = np.searchsorted(dirs, dir)
coords.append(dirCoord)
if ant != None:
antCoord = np.searchsorted(ants, ant)
coords.append(antCoord)
freqCoord = np.searchsorted(freqs, freq)
timeCoord = np.searchsorted(times, time)
vals[tuple(coords)][np.ix_(freqCoord,timeCoord)] = val.T
weights[tuple(coords)][np.ix_(freqCoord,timeCoord)] = 1
#close the parmDB
instrumentdb = 0
vals = np.nan_to_num(vals) # replace nans with 0 (flagged later)
if solType == '*RotationAngle':
np.putmask(weights, vals == 0., 0) # flag where val=0
h5parmDB.makeSoltab(solset, 'rotation', axesNames=['dir','ant','freq','time'], \
axesVals=[dirs,ants,freqs,times], vals=vals, weights=weights, parmdbType=', '.join(list(ptype)))
elif solType == '*ScalarPhase':
np.putmask(weights, vals == 0., 0)
h5parmDB.makeSoltab(solset, 'scalarphase', axesNames=['dir','ant','freq','time'], \
axesVals=[dirs,ants,freqs,times], vals=vals, weights=weights, parmdbType=', '.join(list(ptype)))
elif solType == 'Clock':
np.putmask(weights, vals == 0., 0)
h5parmDB.makeSoltab(solset, 'clock', axesNames=['ant','freq','time'], \
axesVals=[ants,freqs,times], vals=vals, weights=weights, parmdbType=', '.join(list(ptype)))
elif solType == 'TEC':
np.putmask(weights, vals == 0., 0)
h5parmDB.makeSoltab(solset, 'tec', axesNames=['dir','ant','freq','time'], \
axesVals=[dirs,ants,freqs,times], vals=vals, weights=weights, parmdbType=', '.join(list(ptype)))
elif solType == '*Gain:*:Real' or solType == '*Gain:*:Ampl':
np.putmask(vals, vals == 0., 1) # nans end up into 1s (as BBS output, flagged next line)
np.putmask(weights, vals == 1., 0) # flag where val=1
h5parmDB.makeSoltab(solset, 'amplitude', axesNames=['pol','dir','ant','freq','time'], \
axesVals=[pols,dirs,ants,freqs,times], vals=vals, weights=weights, parmdbType=', '.join(list(ptype)))
elif solType == '*Gain:*:Imag' or solType == '*Gain:*:Phase':
#np.putmask(weights, vals == 0., 0) # TODO: NDPPP bug which put at 0 the reference antenna phase
h5parmDB.makeSoltab(solset, 'phase', axesNames=['pol','dir','ant','freq','time'], \
axesVals=[pols,dirs,ants,freqs,times], vals=vals, weights=weights, parmdbType=', '.join(list(ptype)))
antennaTable = pt.table(antennaFile, ack=False)
antennaNames = antennaTable.getcol('NAME')
antennaPositions = antennaTable.getcol('POSITION')
antennaTable.close()
antennaTable = solset._f_get_child('antenna')
antennaTable.append(zip(*(antennaNames,antennaPositions)))
fieldTable = pt.table(fieldFile, ack=False)
phaseDir = fieldTable.getcol('PHASE_DIR')
pointing = phaseDir[0, 0, :]
fieldTable.close()
sourceTable = solset._f_get_child('source')
# add the field centre, that is also the direction for Gain and CommonRotationAngle
sourceTable.append([('pointing',pointing)])
dirs = []
for tab in solset._v_children:
c = solset._f_getChild(tab)
if c._v_name != 'antenna' and c._v_name != 'source':
dirs.extend(list(set(c.dir)))
# remove duplicates
dirs = list(set(dirs))
# remove any pointing (already in the table)
if 'pointing' in dirs:
dirs.remove('pointing')
if dirs != []:
sourceFile = skydbFile + '/SOURCES'
src_table = pt.table(sourceFile, ack=False)
sub_tables = src_table.getsubtables()
vals = []
ra = dec = np.nan
has_patches_subtable = False
for sub_table in sub_tables:
if 'PATCHES' in sub_table:
has_patches_subtable = True
if has_patches_subtable:
# Read values from PATCHES subtable
src_table.close()
sourceFile = skydbFile + '/SOURCES/PATCHES'
src_table = pt.table(sourceFile, ack=False)
patch_names = src_table.getcol('PATCHNAME')
patch_ras = src_table.getcol('RA')
patch_decs = src_table.getcol('DEC')
for source in dirs:
try:
patch_indx = patch_names.index(source)
ra = patch_ras[patch_indx]
dec = patch_decs[patch_indx]
except ValueError:
ra = np.nan
dec = np.nan
vals.append([ra, dec])
src_table.close()
else:
# Try to read default values from parmdb instead
skydb = pdb.parmdb(skydbFile)
vals = []
ra = dec = np.nan
for source in dirs:
try:
ra = skydb.getDefValues('Ra:' + source)['Ra:' + source][0][0]
dec = skydb.getDefValues('Dec:' + source)['Dec:' + source][0][0]
except KeyError:
# Source not found in skymodel parmdb, try to find components
ra = np.array(skydb.getDefValues('Ra:*' + source + '*').values())
dec = np.array(skydb.getDefValues('Dec:*' + source + '*').values())
if len(ra) == 0 or len(dec) == 0:
ra = np.nan
dec = np.nan
else:
ra = ra.mean()
dec = dec.mean()
vals.append([ra, dec])
sourceTable.append(zip(*(dirs,vals)))
solsetname = solset._v_name
# close the hdf5-file
h5parmDB.close()
return solsetname
if len(ms_list) == 0:
print('No measurement sets found in input directory. They must end '
'in .MS, .ms, or .ms.peeled')
sys.exit()
logfilename = options.outdir + '/ion_apply.log'
init_logger(logfilename, debug=options.verbose)
log = logging.getLogger("Main")
log.info('Exporting screens...')
for ms in ms_list:
# Export screens to parmdbs
os.system('H5parm_exporter.py {0} {1} -r ion -s {2} -i {3} >> {4} '
'2>&1'.format(h5, ms, solset, options.parmdb, logfilename))
out_parmdb = 'ion_' + options.parmdb
log.info('Screens exported to {0}'.format(out_parmdb))
# Calibrate
log.info('Calibrating and applying screens...')
workers=Pool(processes=min(len(ms_list), options.ncores))
workers.map(calibrate, zip(ms_list, out_parmdb))
# Clip high data amplitudes
H = h5parm(h5)
station_selection = H.getAnt(solset).keys()
log.info('Clipping CORRECTED_DATA amplitudes at {0} Jy...'.format(options.threshold))
clip(ms_list, station_selection, threshold=options.threshold)
log.info('TEC screen application complete.')
################################
# check input
if len(args.h5parmFiles) == 1 and (',' in args.h5parmFiles[0] or
('[' in args.h5parmFiles[0] and
']' in args.h5parmFiles[0])):
# Treat input as a string with comma-separated values
args.h5parmFiles = args.h5parmFiles[0].strip('[]').split(',')
args.h5parmFiles = [f.strip() for f in args.h5parmFiles]
# read all tables
insolset = args.insolset
if args.insoltab is None:
# use all soltabs, find out names from first h5parm
h5 = h5parm(args.h5parmFiles[0], readonly=True)
solset = h5.getSolset(insolset)
insoltabs = solset.getSoltabNames()
h5.close()
if len(insoltabs) == 0:
logging.critical('No soltabs found.')
sys.exit()
else:
insoltabs = args.insoltab.split(',')
# open input
h5s = []
for h5parmFile in args.h5parmFiles:
h5 = h5parm(h5parmFile.replace("'",""), readonly=True)
h5s.append(h5)
