def main(options):
ms = options.ms
if ms == '':
logging.error('You have to specify an input MS, use -h for help')
return
cols = options.cols
incol = options.incol
t = pt.table(ms, readonly=False, ack=False)
for col in cols.split(','):
if col not in t.colnames():
logging.info('Adding the output column '+col+' to '+ms+'.')
if incol == '':
# prepare col metadata
cd = t.getcoldesc('DATA')
coldmi = t.getdminfo('DATA')
if options.dysco:
cd['dataManagerType'] = 'DyscoStMan'
cd['dataManagerGroup'] = 'DyscoData'
coldmi = {'NAME': col,'SEQNR': 3,'SPEC': {'dataBitCount': 10,'distribution': 'TruncatedGaussian','distributionTruncation': 2.5,'normalization': 'AF','studentTNu': 0.0,'weightBitCount': 12},'TYPE': 'DyscoStMan'}
# not as performing as standard DATA
else:
coldmi["NAME"] = col
# cd['dataManagerType'] = 'StandardStMan'
# cd['dataManagerGroup'] = 'SSMVar'
# coldmi = {'NAME': col,'SEQNR': 0,'SPEC': {'ActualCacheSize': 2,'BUCKETSIZE': 32768,'IndexLength': 799832,'PERSCACHESIZE': 2},'TYPE': 'StandardStMan'}
cd['comment'] = 'Added by addcol2ms'
t.addcols(pt.makecoldesc(col, cd), coldmi)
# if non dysco is done by default
if options.dysco:
logging.warning('Setting '+col+' = 0')
pt.taql("update $t set "+col+"=0")
else:
# prepare col metadata
coldmi = t.getdminfo(incol)
coldmi['NAME'] = col
cd = t.getcoldesc(incol)
cd['comment'] = 'Added by addcol2ms'
t.addcols(pt.makecoldesc(col, cd), coldmi)
logging.warning('Setting '+col+' = '+incol)
pt.taql("update $t set "+col+"="+incol)
else:
logging.warning('Column '+col+' already exists.')
t.close()
def iter_antenna(self, antennas=None):
"""
Iterator to get all visibilities of each antenna
For GFLAG, GDATA and GWEIGHTS it returns arrays of Ntimes x Nbl x Nfreqs x Npol
"""
ms = self.ms # to use the "$" in taql
for ant_id, ant_name in enumerate(self.get_antennas()):
if antennas is not None and ant_name not in antennas: continue
logging.info('Workign on antenna: %s', ant_name)
yield ant_id, ant_name, taql('select TIME, GAGGR(FLAG) as GFLAG, ANTENNA1, ANTENNA2, GAGGR(%s) as GDATA, GAGGR(%s) as GWEIGHT \
from $ms \
where (ANTENNA1=%i or ANTENNA2=%i) and (ANTENNA1 != ANTENNA2) \
groupby TIME' \
% (self.dcolname, self.wcolname, ant_id, ant_id) )
def run( soltab, ms, inverse=False, useElementResponse=True, useArrayFactor=True, useChanFreq=True ):
"""
Generic unspecified step for easy expansion.
Parameters
----------
opt1 : float
Is a mandatory parameter.
"""
# load specific libs
import numpy as np
import casacore.tables as pt
from lofar.stationresponse import stationresponse
sr = stationresponse(ms, inverse, useElementResponse, useArrayFactor, useChanFreq)
numants = pt.taql('select gcount(*) as numants from '+ms+'::ANTENNA').getcol('numants')[0]
times = soltab.getAxisValues('time')
for vals, coord, selection in soltab.getValuesIter(returnAxes=['ant','time','pol','freq'], weight=False):
vals = reorderAxes( vals, soltab.getAxesNames(), ['ant','time','freq','pol'] )
for stationnum in range(numants):
logging.debug('Working on station number %i' % stationnum)
for itime, time in enumerate(times):
beam = sr.evaluateStation(time=time, station=stationnum)
# Reshape from [nfreq, 2, 2] to [nfreq, 4]
beam = beam.reshape(beam.shape[0], 4)
if soltab.getAxisLen('pol') == 2:
beam = beam[:,[0,3]] # get only XX and YY
if soltab.getType() == 'amplitude':
vals[stationnum, itime, :, :] = np.abs(beam)
elif soltab.getType() == 'phase':
vals[stationnum, itime, :, :] = np.angle(beam)
else:
logging.error('Beam prediction work only for amplitude/phase solution tables.')
return 1
vals = reorderAxes( vals, ['ant','time','freq','pol'], [ax for ax in soltab.getAxesNames() if ax in ['ant','time','freq','pol']] )
soltab.setValues(vals, selection)
return 0
def test_subset(self):
"""Create a subset."""
c1 = makescacoldesc("coli", 0)
c2 = makescacoldesc("cold", 0.)
c3 = makescacoldesc("cols", "")
c4 = makescacoldesc("colb", True)
c5 = makescacoldesc("colc", 0. + 0j)
c6 = makearrcoldesc("colarr", 0.)
t = table("ttable.py_tmp.tab1", maketabdesc((c1, c2, c3, c4, c5,
c6)), ack=False)
t1 = t.query('coli >0', sortlist='coli desc', columns='coli,cold')
querycols = t1.colnames()
t1 = taql('select coli,cold from $t where coli>0 order by coli desc')
taqlcol = t1.colnames()
self.assertEqual(querycols, taqlcol)
t1.close()
t.close()
tabledelete("ttable.py_tmp.tab1")
def get_antennas(self):
return taql('select NAME from %s/ANTENNA' % (self.ms_files[0]) ).getcol('NAME')
# Get the azimuth and elevation of Jupiter from Dwingeloo at a given time.
# Usage: python azel-taql.py
from casacore.tables import taql
print taql('calc meas.azel("Jupiter",2016-06-28 19:54,"DWL") deg')
sys.exit(0)
msfile = msfile[0]
if not os.path.exists(msfile):
logging.error("Cannot find MS file {}.".format(msfile))
sys.exit(1)
# open input/output MS
ms = pt.table(msfile, readonly=False, ack=False)
with pt.table(msfile + '::SPECTRAL_WINDOW', ack=False) as freqtab:
freq = freqtab.getcol('REF_FREQUENCY')[0]
freqpersample = np.mean(freqtab.getcol('RESOLUTION'))
timepersample = ms.getcell('INTERVAL', 0)
# get info on all baselines
with pt.taql(
"SELECT ANTENNA1,ANTENNA2,sqrt(sumsqr(UVW)),GCOUNT() FROM $ms GROUPBY ANTENNA1,ANTENNA2"
) as BL:
ants1, ants2 = BL.getcol('ANTENNA1'), BL.getcol('ANTENNA2')
dists = BL.getcol('Col_3') / 1e3 # baseleline length in km
n_t = BL.getcol('Col_4')[0] # number of timesteps
n_bl = len(ants1)
# check if ms is time-ordered
times = ms.getcol('TIME_CENTROID')
if not all(np.diff(times) >= 0):
logging.critical('This code cannot handle MS that are not time-sorted.')
sys.exit(1)
del times
# create column to smooth
addcol(ms, options.incol, options.outcol)
def read_main_table(infile, subsel=0, ignore=None, chunks=(400, 200, 100, 2)):
if ignore is None: ignore = []
# select just the specified ddi
tb_tool = tables.taql('select * from %s where DATA_DESC_ID = %i' %
(infile, subsel))
if tb_tool.nrows() == 0:
tb_tool.close()
return xarray.Dataset()
# main table uses time x (antenna1,antenna2)
ant1, ant2 = tb_tool.getcol('ANTENNA1', 0,
-1), tb_tool.getcol('ANTENNA2', 0, -1)
baselines = np.array([
str(ll[0]).zfill(3) + '_' + str(ll[1]).zfill(3)
for ll in np.unique(np.hstack([ant1[:, None], ant2[:, None]]), axis=0)
])
dims, cols = ['time', 'baseline', 'chan', 'pol'], tb_tool.colnames()
tvars, mcoords, xds = {}, {}, xarray.Dataset()
cshapes = [
np.array(tb_tool.getcell(col, 0)).shape for col in cols
if tb_tool.iscelldefined(col, 0)
]
chan_cnt, pol_cnt = [(cc[0], cc[1]) for cc in cshapes if len(cc) == 2][0]
ts_tb = tables.taql(
'select DISTINCT TIME from %s where DATA_DESC_ID = %i' %
(infile, subsel))
utimes = np.unique(ts_tb.getcol('TIME', 0, -1))
tol = np.diff(
utimes).min() / 4 # add a tol around the time ranges returned by taql
ts_tb.close()
# loop over time chunks
for tc in range(0, len(utimes), chunks[0]):
times = (utimes[tc] - tol,
utimes[min(len(utimes) - 1, tc + chunks[0] - 1)] + tol)
ctlen = min(len(utimes), tc + chunks[0]) - tc # chunk time length
bvars = {}
# loop over baseline chunks
for bc in range(0, len(baselines), chunks[1]):
blines = (baselines[bc],
baselines[min(len(baselines) - 1, bc + chunks[1] - 1)])
cblen = min(len(baselines) - bc, chunks[1])
# read the specified chunk of data
#def read_chunk(infile, subsel, times, blines, chans, pols):
dtql = 'DATA_DESC_ID = %i' % subsel
ttql = 'TIME BETWEEN %f and %f' % times
atql = 'ANTENNA1 BETWEEN %i and %i' % (int(
blines[0].split('_')[0]), int(blines[1].split('_')[0]))
ts_taql = 'select * from %s where %s AND %s AND %s' % (
infile, dtql, ttql, atql)
ts_tb = tables.taql(ts_taql)
tidxs = np.searchsorted(utimes, ts_tb.getcol('TIME', 0, -1)) - tc
ts_ant1, ts_ant2 = ts_tb.getcol('ANTENNA1', 0, -1), ts_tb.getcol(
'ANTENNA2', 0, -1)
ts_bases = [
str(ll[0]).zfill(3) + '_' + str(ll[1]).zfill(3)
for ll in np.hstack([ts_ant1[:, None], ts_ant2[:, None]])
]
bidxs = np.searchsorted(baselines, ts_bases) - bc
# some antenna 2's will be out of bounds for this chunk, store rows that are in bounds
didxs = np.where((bidxs >= 0)
& (bidxs < min(chunks[1],
len(baselines) - bc)))[0]
# loop over each column and create delayed dask arrays
for col in cols:
if (col in ignore +
['TIME']) or (not tb_tool.iscelldefined(col, 0)):
continue
if col not in bvars: bvars[col] = []
cdata = tb_tool.getcol(col, 0, 1)[0]
if len(cdata.shape) == 0:
delayed_array = dask.delayed(read_col_chunk)(
ts_taql, col, (ctlen, cblen), tidxs, bidxs, didxs,
None, None)
bvars[col] += [
dask.array.from_delayed(delayed_array, (ctlen, cblen),
cdata.dtype)
]
elif col == 'UVW':
delayed_array = dask.delayed(read_col_chunk)(
ts_taql, col, (ctlen, cblen, 3), tidxs, bidxs, didxs,
None, None)
bvars[col] += [
dask.array.from_delayed(delayed_array,
(ctlen, cblen, 3), cdata.dtype)
]
elif len(cdata.shape) == 1:
pol_list = []
dd = 2 if cdata.shape == chan_cnt else 3
for pc in range(0, cdata.shape[0], chunks[dd]):
pols = (pc, min(cdata.shape[0], pc + chunks[dd]) - 1)
cshape = (
ctlen,
cblen,
) + (pols[1] - pols[0] + 1, )
delayed_array = dask.delayed(read_col_chunk)(
ts_taql, col, cshape, tidxs, bidxs, didxs, pols,
None)
pol_list += [
dask.array.from_delayed(delayed_array, cshape,
cdata.dtype)
]
bvars[col] += [dask.array.concatenate(pol_list, axis=2)]
elif len(cdata.shape) == 2:
chan_list = []
for cc in range(0, cdata.shape[0], chunks[2]):
chans = (cc, min(cdata.shape[0], cc + chunks[2]) - 1)
pol_list = []
for pc in range(0, cdata.shape[1], chunks[3]):
pols = (pc,
min(cdata.shape[1], pc + chunks[3]) - 1)
cshape = (
ctlen,
cblen,
) + (chans[1] - chans[0] + 1,
pols[1] - pols[0] + 1)
delayed_array = dask.delayed(read_col_chunk)(
ts_taql, col, cshape, tidxs, bidxs, didxs,
chans, pols)
pol_list += [
dask.array.from_delayed(
delayed_array, cshape, cdata.dtype)
]
chan_list += [dask.array.concatenate(pol_list, axis=3)]
bvars[col] += [dask.array.concatenate(chan_list, axis=2)]
ts_tb.close()
# now concat all the dask chunks from each baseline
for kk in bvars.keys():
if len(bvars[kk]) == 0: continue
if kk not in tvars: tvars[kk] = []
tvars[kk] += [dask.array.concatenate(bvars[kk], axis=1)]
# now concat all the dask chunks from each time to make the xds
mvars = {}
for kk in tvars.keys():
if kk == 'UVW':
mvars[kk.upper()] = xarray.DataArray(dask.array.concatenate(
tvars[kk], axis=0),
dims=dims[:2] + ['uvw_index'])
elif len(tvars[kk][0].shape) == 3 and (tvars[kk][0].shape[-1]
== pol_cnt):
mvars[kk.upper()] = xarray.DataArray(dask.array.concatenate(
tvars[kk], axis=0),
dims=dims[:2] + ['pol'])
elif len(tvars[kk][0].shape) == 3 and (tvars[kk][0].shape[-1]
== chan_cnt):
mvars[kk.upper()] = xarray.DataArray(dask.array.concatenate(
tvars[kk], axis=0),
dims=dims[:2] + ['chan'])
else:
mvars[kk.upper()] = xarray.DataArray(
dask.array.concatenate(tvars[kk], axis=0),
dims=dims[:len(tvars[kk][0].shape)])
mcoords['time'] = xarray.DataArray(convert_time(utimes), dims=['time'])
mcoords['baseline'] = xarray.DataArray(np.arange(len(baselines)),
dims=['baseline'])
xds = xarray.Dataset(mvars, coords=mcoords)
tb_tool.close()
return xds
#!/usr/bin/python
# usage: plot_weights.py xxx.MS
import os, sys
#sys.path = ['/home/dijkema/opt/python-casacore/lib/python2.7/site-packages/']
import numpy as np
import matplotlib.cm as cm
import matplotlib.pyplot as plt
from casacore.tables import taql
ms = sys.argv[1]
print "selecting on chan"
t=taql('select ANTENNA, gmeans(DATA[FLAG]) as DATA, gmeans(WEIGHT_SPECTRUM[FLAG]) as WEIGHT from [[ SELECT ANTENNA1 AS ANTENNA, DATA, WEIGHT_SPECTRUM, FLAG from '+ms+'], [SELECT ANTENNA2 AS ANTENNA, DATA, WEIGHT_SPECTRUM, FLAG from '+ms+']] group by ANTENNA')
#weights = np.average(np.average(t.getcol('WEIGHT'),axis=0),axis=1)
weights = t.getcol('WEIGHT')[:,:,0]
data = np.abs(t.getcol('DATA')[:,:,0])
fig, ax1 = plt.subplots()
colors = iter(cm.rainbow(np.linspace(0, 1, len(weights))))
#ax1.set_yscale('log')
for w, c in zip(weights, colors):
ax1.scatter(xrange(len(w)),w,color=c,marker='.')
ax1.set_xlim(xmin=0,xmax=len(w))
ax1.set_ylim(ymin=np.min(weights),ymax=np.max(weights))
ax1.set_xlabel('Channel')
ax1.set_ylabel('Weights')
plt.legend()
plt.savefig('weightVSchan.png', bbox_inches='tight')
fig, ax1 = plt.subplots()
def reweight(MSh, mode):
# get data per antenna
var_antenna = {}
med_antenna = {}
for ant_id, ant_name, ms_ant in MSh.iter_antenna():
with Timer('Get data'):
data = ms_ant.getcol('GDATA') # axes: time, ant, freq, pol
flags = ms_ant.getcol('GFLAG') # axes: time, ant, freq, pol
# put flagged data to NaNs
data[flags] = np.nan
# if completely flagged set variance to 1 and continue
if np.all(flags):
var_antenna[ant_id] = None
med_antenna[ant_id] = None
continue
with Timer('Prepare data'):
# data column is updated subtracting adjacent channels
if mode == 'subchan':
data_shifted_l = np.roll(data, -1, axis=2)
data_shifted_r = np.roll(data, +1, axis=2)
# if only 2 freq it's aleady ok, subtracting one from the other
if data.shape[2] > 2:
data_shifted_l[:,:,-1,:] = data_shifted_l[:,:,-3,:] # last chan uses the one but last
data_shifted_r[:,:,0,:] = data_shifted_r[:,:,2,:] # first chan uses third
# get the "best" shift, either on the right or left. This is to avoid propagating bad channels (e.g. with RFI)
ratio_l = np.nanvar(data_shifted_l, axis=(0,1,3))/np.nanmean(data_shifted_l, axis=(0,1,3))
ratio_l[ np.isnan(ratio_l) ] = np.inf
ratio_r = np.nanvar(data_shifted_r, axis=(0,1,3))/np.nanmean(data_shifted_r, axis=(0,1,3))
ratio_r[ np.isnan(ratio_r) ] = np.inf
data = np.where( ( ratio_l < ratio_r )[np.newaxis,np.newaxis,:,np.newaxis], data - data_shifted_l, data - data_shifted_r)
# data column is updated subtracting adjacent times
if mode == 'subtime':
data_shifted_l = np.roll(data, -1, axis=0)
data_shifted_r = np.roll(data, +1, axis=0)
# if only 2 freq it's aleady ok, subtracting one from the other
if data.shape[0] > 2:
data_shifted_l[-1,:,:,:] = data_shifted_l[-3,:,:,:] # last timeslot uses the one but last
data_shifted_r[0,:,:,:] = data_shifted_r[2,:,:,:] # first timeslot uses third
# get the "best" shift, either on the right or left. This is to avoid propagating bad channels (e.g. with RFI)
ratio_l = np.nanvar(data_shifted_l, axis=(1,2,3))/np.nanmean(data_shifted_l, axis=(1,2,3))
ratio_l[ np.isnan(ratio_l) ] = np.inf
ratio_r = np.nanvar(data_shifted_r, axis=(1,2,3))/np.nanmean(data_shifted_r, axis=(1,2,3))
ratio_r[ np.isnan(ratio_r) ] = np.inf
data = np.where( ( ratio_l < ratio_r )[:,np.newaxis,np.newaxis,np.newaxis], data - data_shifted_l, data - data_shifted_r)
# use residual data, nothing to do here
elif mode == 'residual':
pass
with Timer('Calc variances'):
# find mean/variance per time/freq for each antenna
med_freqs = np.abs( np.nanmean( data, axis=(1,2) )**2 ) # time x pol
med_times = np.abs( np.nanmean( data, axis=(0,1) )**2 ) # freq x pol
med_antenna[ant_id] = med_freqs[:, np.newaxis]+med_times # sum of the time/freq mean - axes: time,freq,pol
var_freqs = np.nanvar( data, axis=(1,2) ) # time x pol
var_times = np.nanvar( data, axis=(0,1) ) # freq x pol
var_antenna[ant_id] = var_freqs[:, np.newaxis]+var_times # sum of the time/freq variances - axes: time,freq,pol
# reconstruct BL weights from antenna variance
for ms_bl in MSh.ms.iter(["ANTENNA1","ANTENNA2"]):
ant_id1 = ms_bl.getcol('ANTENNA1')[0]
ant_id2 = ms_bl.getcol('ANTENNA2')[0]
if var_antenna[ant_id1] is None or var_antenna[ant_id2] is None: continue
# print '### BL: %i - %i' % (ant_id1, ant_id2)
# print var_antenna[ant_id1]*med_antenna[ant_id2]
# print ''
# print var_antenna[ant_id2]*med_antenna[ant_id1]
# print ''
# print var_antenna[ant_id1]*var_antenna[ant_id2]
w = 1./( var_antenna[ant_id1]*med_antenna[ant_id2] + var_antenna[ant_id2]*med_antenna[ant_id1] \
+ var_antenna[ant_id1]*var_antenna[ant_id2] )
w -= np.nanmedian(w) # TEST: REMOVE MEDIAN?
f = ms_bl.getcol('FLAG')
# find how many unflagged weights are nans
ntoflag = np.count_nonzero(np.isnan(w[~f]))
logging.debug( 'BL: %i - %i: created %i new flags (%f%%)' % ( ant_id1, ant_id2, ntoflag, (100.*ntoflag)/np.size(w) ) )
ms_bl.putcol(MSh.wcolname, w)
ms_bl.flush()
# flag weights that are nans
taql('update $ms_bl set FLAG[isnan(WEIGHT_SPECTRUM)]=True, WEIGHT_SPECTRUM[isnan(WEIGHT_SPECTRUM)]=0')
ms_bl.flush()
def get_elev(self):
# TODO: fix if multiple time MSs passed
ms_avgbl = taql('SELECT TIME, MEANS(GAGGR(MSCAL.AZEL1()[1]), 0) AS ELEV FROM %s GROUPBY TIME' %(self.ms_files[0]))
return ms_avgbl.getcol('ELEV')
st = ss.getSoltab('phase000')
ax_freq_temp = st.getAxisValues('freq')
if len(ax_freq_temp) > len_freq_old:
# Longer freq axis meas a shorter solution interval was used.
ax_freq = ax_freq_temp
len_freq_old = len(ax_freq)
name_freq = ih5
fh5.close()
print('Fastest frequency axis taken from {:s} with a solution interval of {:f} Hz.'.format(name_freq, ax_freq[1]-ax_freq[0]))
if 'pol' in axes_new:
phases = np.zeros((len(polarizations), 1, len(antennas), len(ax_freq), len(ax_time)))
else:
phases = np.zeros((1, len(antennas), len(ax_freq), len(ax_time)))
elif convert_tec and 'tec' in args.soltab2merge:
print('Determining frequency grid...')
ff = ct.taql('SELECT CHAN_FREQ, CHAN_WIDTH FROM ' + ms_first + '::SPECTRAL_WINDOW')
freq_first = ff.getcol('CHAN_FREQ')[0][0]
freq_spacing = ff.getcol('CHAN_WIDTH')[0][0]
ff.close()
fl = ct.taql('SELECT CHAN_FREQ, CHAN_WIDTH FROM ' + ms_last + '::SPECTRAL_WINDOW')
freq_last = fl.getcol('CHAN_FREQ')[0][0]
print(freq_first, freq_last, freq_spacing)
ax_freq = np.arange(freq_first, freq_last + freq_spacing, freq_spacing)
phases = np.zeros((1, 1, len(antennas), len(ax_freq), len(ax_time)))
print('Frequency axis taken Measurement Sets with a solution interval of {:f} Hz.'.format(ax_freq[1]-ax_freq[0]))
elif not convert_tec:
phases = np.zeros(vals_reordered.shape)
h5out = h5parm.h5parm(args.h5out, readonly=False)
if args.append_to_solset:
#############################
# 3. Set-up directory structure ans split
MSs = lib_ms.AllMSs(glob.glob(data_dir + '/*MS'), s)
numberOfMSs = len(MSs.getListObj())
if not os.path.exists('cals'): os.mkdir('cals')
if not os.path.exists('tgts'): os.mkdir('tgts')
for msID, MS in enumerate(MSs.getListObj()):
pathMS = MS.pathMS
logger.info("- Split-up of original MS '%s' (msID: %02i) -" %
(pathMS, msID))
scanIDs = tables.taql("select distinct SCAN_NUMBER from $pathMS").getcol(
"SCAN_NUMBER")
numberOfScans = len(scanIDs)
# Create temporary paths for the sub-MSs.
for i, scanID in enumerate(scanIDs):
pathMSscan = "msID%02i_scanID%03i.MS" % (msID, scanID)
if os.path.exists(pathMSscan): continue
logger.info("MS %02i/%02i: %03i/%03i - Creating MS: %s" %
(msID + 1, numberOfMSs, i + 1, numberOfScans, pathMSscan))
tables.taql(
"SELECT from $pathMS where SCAN_NUMBER = $scanID giving $pathMSscan as plain"
)
#############################
# 4. Fix MS
MSs = lib_ms.AllMSs(glob.glob('msID*_scanID*MS'), s)
def scan_ms(self):
"""
Scans input MS and stores info
"""
# Get time info
tab = pt.table(self.ms_filename, ack=False)
if self.starttime is None:
self.starttime = np.min(tab.getcol('TIME'))
else:
valid_times = np.where(tab.getcol('TIME') >= self.starttime)[0]
if len(valid_times) == 0:
self.log.critical('Start time of {0} is greater than the last time in the MS! '
'Exiting!'.format(self.starttime))
sys.exit(1)
self.starttime = tab.getcol('TIME')[valid_times[0]]
# DPPP takes ceil(startTimeParset - startTimeMS), so ensure that our start time is
# slightly less than the true one (if it's slightly larger, DPPP sets the first
# time to the next time, skipping the first time slot)
self.starttime -= 0.1
if self.starttime > np.min(tab.getcol('TIME')):
self.startsat_startofms = False
else:
self.startsat_startofms = True
if self.endtime is None:
self.endtime = np.max(tab.getcol('TIME'))
else:
valid_times = np.where(tab.getcol('TIME') <= self.endtime)[0]
if len(valid_times) == 0:
self.log.critical('End time of {0} is less than the first time in the MS! '
'Exiting!'.format(self.endtime))
sys.exit(1)
self.endtime = tab.getcol('TIME')[valid_times[-1]]
if self.endtime < np.max(tab.getcol('TIME')):
self.goesto_endofms = False
else:
self.goesto_endofms = True
self.timepersample = tab.getcell('EXPOSURE', 0)
self.numsamples = int(np.ceil((self.endtime - self.starttime) / self.timepersample))
tab.close()
# Get frequency info
sw = pt.table(self.ms_filename+'::SPECTRAL_WINDOW', ack=False)
self.referencefreq = sw.col('REF_FREQUENCY')[0]
self.startfreq = np.min(sw.col('CHAN_FREQ')[0])
self.endfreq = np.max(sw.col('CHAN_FREQ')[0])
self.numchannels = sw.col('NUM_CHAN')[0]
self.channelwidth = sw.col('CHAN_WIDTH')[0][0]
sw.close()
# Get pointing info
obs = pt.table(self.ms_filename+'::FIELD', ack=False)
self.ra = np.degrees(float(obs.col('REFERENCE_DIR')[0][0][0]))
if self.ra < 0.:
self.ra = 360.0 + (self.ra)
self.dec = np.degrees(float(obs.col('REFERENCE_DIR')[0][0][1]))
obs.close()
# Get station names and diameter
ant = pt.table(self.ms_filename+'::ANTENNA', ack=False)
self.stations = ant.col('NAME')[:]
self.diam = float(ant.col('DISH_DIAMETER')[0])
if 'HBA' in self.stations[0]:
self.antenna = 'HBA'
elif 'LBA' in self.stations[0]:
self.antenna = 'LBA'
else:
self.log.warning('Antenna type not recognized (only LBA and HBA data '
'are supported at this time)')
ant.close()
# Find mean elevation and FOV
el_values = pt.taql("SELECT mscal.azel1()[1] AS el from "
+ self.ms_filename + " limit ::10000").getcol("el")
self.mean_el_rad = np.mean(el_values)
parameter_set['RA_DEC_Flux_idx'] = [1, 3, 7]
else:
RuntimeError('Unknown catalog: {}'.format(catalog))
data_root_path = os.environ['DATA_ROOT_PATH']
basefile_name = 'RX42_SB100-109.2ch10s'
# basefile_name = 'BOOTES24_SB180-189.2ch8s_SIM'
catalog_basefile_name = parameter_set['catalog_basefile_name']
ms_file_name = data_root_path + basefile_name + '.ms'
catalog_file = data_root_path + catalog_basefile_name + file_ext
npz_catalog_name = data_root_path + catalog_basefile_name + '.npz'
# pointing direction in radian
pointingDirection = casa_tables.taql(
'select REFERENCE_DIR from {msFile}::FIELD'.format(msFile=ms_file_name)
).getcol('REFERENCE_DIR').squeeze()
M = J2000_to_UVW_operator(*pointingDirection)
# index in the fits table where RA, DEC and Flux are
RA_DEC_Flux_idx = parameter_set['RA_DEC_Flux_idx']
with fits.open(catalog_file) as handle:
# print out a few information about the HDU file and the data column description
handle.info()
data_description = handle[1].header
data_description
# the actual data
data = handle[1].data
# extract data <- verify with the data_description that the correct data is extracted
# RA and DEC are in J2000 coordinate (in degrees)
ax = fig.add_subplot(111)
im = ax.imshow(screen[..., -1], origin='lower', extent=[ra_max, ra_min, dec_min, dec_max])
ax.scatter(RA, DEC, marker='x', color='r')
ax.set_xlabel('Right ascension'); ax.set_ylabel('Declination')
fig.colorbar(im)
fig.savefig(antname + '_{:03d}.png'.format(itstep))
del fig
show()
'''
return names.index(antname), screen
ms = sys.argv[1]
h5p = sys.argv[2]
t = ct.taql('SELECT NAME FROM {ms:s}::ANTENNA'.format(ms=ms))
names = t.getcol('NAME')
t.close()
t = ct.taql('SELECT REFERENCE_DIR FROM {ms:s}::FIELD'.format(ms=ms))
phasecenter = t.getcol('REFERENCE_DIR').squeeze()
print(phasecenter)
# Avoid negative CRVAL1
if np.rad2deg(phasecenter[0]) < 0:
phasecenter[0] = phasecenter[0] + (2. * np.pi)
t.close()
h5 = h5parm.h5parm(h5p)
ss = h5.getSolset('sol000')
st = ss.getSoltab('amplitude000')
time = st.getAxisValues('time')
stime = time[0]
def get_freqs(self):
# TODO: check if there is a smarter way to do it with concat.MS
freqs = []
for ms_file in self.ms_files:
freqs += list( taql('SELECT CHAN_FREQ FROM %s/SPECTRAL_WINDOW' % ms_file)[0]['CHAN_FREQ'] * 1e-6 ) # in MHz
return freqs
def test_msutil(self):
"""Testing msutil."""
datacoldesc = makearrcoldesc("DATA", 0., ndim=2, shape=[20, 4])
ms = default_ms("tabtemp", maketabdesc((datacoldesc)))
ms.close()
spw = table("tabtemp/SPECTRAL_WINDOW", readonly=False)
spw.addrows()
spw.putcell('NUM_CHAN', 0, 20)
t = table("tabtemp", readonly=False)
print(t.colnames())
addImagingColumns("tabtemp")
self.assertIn('MODEL_DATA', t.colnames())
self.assertIn('CORRECTED_DATA', t.colnames())
self.assertIn('IMAGING_WEIGHT', t.colnames())
removeImagingColumns("tabtemp")
self.assertNotIn('MODEL_DATA', t.colnames())
self.assertNotIn('CORRECTED_DATA', t.colnames())
self.assertNotIn('IMAGING_WEIGHT', t.colnames())
addDerivedMSCal("tabtemp")
self.assertIn('PA1', t.colnames())
self.assertIn('PA2', t.colnames())
self.assertIn('LAST', t.colnames())
self.assertIn('AZEL2', t.colnames())
self.assertIn('AZEL1', t.colnames())
self.assertIn('UVW_J2000', t.colnames())
self.assertIn('LAST1', t.colnames())
self.assertIn('LAST2', t.colnames())
self.assertIn('HA1', t.colnames())
self.assertIn('HA2', t.colnames())
self.assertIn('HA', t.colnames())
removeDerivedMSCal("tabtemp")
self.assertNotIn('PA1', t.colnames())
self.assertNotIn('PA2', t.colnames())
self.assertNotIn('LAST', t.colnames())
self.assertNotIn('AZEL2', t.colnames())
self.assertNotIn('AZEL1', t.colnames())
self.assertNotIn('UVW_J2000', t.colnames())
self.assertNotIn('LAST1', t.colnames())
self.assertNotIn('LAST2', t.colnames())
self.assertNotIn('HA1', t.colnames())
self.assertNotIn('HA2', t.colnames())
self.assertNotIn('HA', t.colnames())
self.assertNotIn('HA', t.colnames())
self.assertNotIn('HA', t.colnames())
taql("SELECT FROM tabtemp where TIME in (SELECT DISTINCT TIME" +
" FROM tabtemp LIMIT 10) GIVING first10.MS AS PLAIN")
taql("SELECT FROM tabtemp where TIME in (SELECT DISTINCT TIME" +
" FROM tabtemp LIMIT 10 OFFSET 10) GIVING second10.MS AS PLAIN")
msconcat(["first10.MS", "second10.MS"], "combined.MS", concatTime=True)
spw.close()
t.close()
tabledelete("tabtemp")
def get_time(self):
# TODO: fix if multiple time MSs passed
ms_avgbl = taql('SELECT TIME FROM %s GROUPBY TIME' %(self.ms_files[0]))
return ms_avgbl.getcol('TIME')
def get_flags_aggr(self):
"""
"""
ms = self.ms_file
return taql('select NTRUES(GAGGR(FLAG), [0, 2]) as FLAG, SHAPE(GAGGR(FLAG)) as N from $ms where ANTENNA1 != ANTENNA2 groupby TIME')
def read_pointing_table(infile, chunks=(100, 100, 20, 20)):
tb_tool = tables.taql('select * from %s' % infile)
if tb_tool.nrows() == 0:
tb_tool.close()
return xarray.Dataset()
# pointing table uses time x antenna_id
baselines = np.unique(tb_tool.getcol('ANTENNA_ID', 0, -1))
dims, cols = ['time', 'antenna_id', 'd2', 'd3'], tb_tool.colnames()
tvars, mcoords, xds = {}, {}, xarray.Dataset()
ts_tb = tables.taql('select DISTINCT TIME from %s' % infile)
utimes = np.unique(ts_tb.getcol('TIME', 0, -1))
tol = np.diff(
utimes).min() / 4 # add a tol around the time ranges returned by taql
ts_tb.close()
# loop over time chunks
for tc in range(0, len(utimes), chunks[0]):
times = (utimes[tc] - tol,
utimes[min(len(utimes) - 1, tc + chunks[0] - 1)] + tol)
ctlen = min(len(utimes), tc + chunks[0]) - tc # chunk time length
bvars = {}
# loop over antenna_id chunks
for bc in range(0, len(baselines), chunks[1]):
blines = (baselines[bc],
baselines[min(len(baselines) - 1, bc + chunks[1] - 1)])
cblen = min(len(baselines) - bc, chunks[1])
# read the specified chunk of data
ttql = 'TIME BETWEEN %f and %f' % times
atql = 'ANTENNA_ID BETWEEN %i and %i' % blines
ts_taql = 'select * from %s where %s AND %s' % (infile, ttql, atql)
ts_tb = tables.taql(ts_taql)
tidxs = np.searchsorted(utimes, ts_tb.getcol('TIME', 0, -1)) - tc
bidxs = np.searchsorted(baselines, ts_tb.getcol(
'ANTENNA_ID', 0, -1)) - bc
didxs = np.arange(len(bidxs))
# loop over each column and create delayed dask arrays
for col in cols:
if (col in ['TIME', 'ANTENNA_ID'
]) or (not tb_tool.iscelldefined(col, 0)):
continue
if col not in bvars: bvars[col] = []
cdata = tb_tool.getcol(col, 0, 1)[0]
if isinstance(cdata, str): cdata = np.array(cdata)
if len(cdata.shape) == 0:
delayed_array = dask.delayed(read_col_chunk)(
ts_taql, col, (ctlen, cblen), tidxs, bidxs, didxs,
None, None)
bvars[col] += [
dask.array.from_delayed(delayed_array, (ctlen, cblen),
cdata.dtype)
]
elif len(cdata.shape) == 2:
d1_list = []
for cc in range(0, cdata.shape[0], chunks[2]):
d1s = (cc, min(cdata.shape[0], cc + chunks[2]) - 1)
d2_list = []
for pc in range(0, cdata.shape[1], chunks[3]):
d2s = (pc, min(cdata.shape[1], pc + chunks[3]) - 1)
cshape = (
ctlen,
cblen,
) + (d1s[1] - d1s[0] + 1, d2s[1] - d2s[0] + 1)
delayed_array = dask.delayed(read_col_chunk)(
ts_taql, col, cshape, tidxs, bidxs, didxs, d1s,
d2s)
d2_list += [
dask.array.from_delayed(
delayed_array, cshape, cdata.dtype)
]
d1_list += [dask.array.concatenate(d2_list, axis=3)]
bvars[col] += [dask.array.concatenate(d1_list, axis=2)]
ts_tb.close()
# now concat all the dask chunks from each baseline
for kk in bvars.keys():
if len(bvars[kk]) == 0: continue
if kk not in tvars: tvars[kk] = []
tvars[kk] += [dask.array.concatenate(bvars[kk], axis=1)]
# now concat all the dask chunks from each time to make the xds
mvars = {}
for kk in tvars.keys():
mvars[kk.upper()] = xarray.DataArray(
dask.array.concatenate(tvars[kk], axis=0),
dims=dims[:len(tvars[kk][0].shape)])
mcoords['time'] = xarray.DataArray(convert_time(utimes), dims=['time'])
mcoords['antenna_id'] = xarray.DataArray(np.arange(len(baselines)),
dims=['antenna_id'])
xds = xarray.Dataset(mvars, coords=mcoords)
tb_tool.close()
return xds
def do_execute(self, code, silent, store_history=True, user_expressions=None,
allow_stdin=False):
ashtml=False
if not silent:
output=""
try:
code=re.sub(ur"([0-9\s\.\)\"])\xb5([A-Za-z])",ur"\1u\2",code) # Tolerate µ as prefix for a unit, substitute it with u
code=re.sub(ur"\u2245",ur"~=",code) # Tolerate ≅ for approximately equal
code=str(code) # Code seems to be unicode, convert to string here
# match the first operation keyword, so that only "select * from (update ..." will yield rows
m=re.match(".*?((?:select)|(?:update)|(?:insert)|(?:delete)|(?:count)|(?:calc)|(?:create table)|(?:insert)|(?:alter table)|(?:show)|(?:help))",code.lower())
if m:
operation=m.group(1)
else:
operation="select"
code="select "+code
t=pt.taql(code)
# Don't display output if code is 'SELECT FROM'
printrows=False
if operation=="select":
# first select has something between "select" and "from"
match = re.match('^.*?select(.*?)from',code, re.IGNORECASE)
if not(match and match.group(1).isspace()):
printrows=True
printcount=True
# Don't display row count in simple calc-like expressions
if operation=="show" or operation=="help" or operation=="calc" or (operation=="select" and not('from' in code.lower())):
printcount=False
if printcount:
ashtml=True
output=self.format_output(t,printrows,printcount,operation,ashtml)
except UnicodeEncodeError as e:
output+="Error: unicode is not supported"
except RuntimeError as e:
myerror=str(e).split('\n')
output=""
m = re.search('parse error at or near position (\d+) ', myerror[1])
if (m):
ashtml=True
pos=int(m.group(1))+22
output+='<pre>'+myerror[0][0:pos]
output+='<span style="background:red;color:white">'+(myerror[0]+' ')[pos]+'</span>'
output+=myerror[0][pos+1:]+'\n'
output+="\n".join(myerror[1:])
#output+=myerror[1]+":"+" "*(21+pos-len(myerror[1]))+"^\n"
output+='</pre>'
else:
output+=myerror[0]+"\n"
output+="\n".join(myerror[1:])
if ashtml:
stream_content={'source': 'TaQL kernel', 'data': {'text/html':output}, 'metadata': {}}
self.send_response(self.iopub_socket, 'display_data', stream_content)
else:
stream_content = {'name': 'stdout', 'text': output}
self.send_response(self.iopub_socket, 'stream', stream_content)
return {'status': 'ok',
# The base class increments the execution count
'execution_count': self.execution_count,
'payload': [],
'user_expressions': {},
}
def describe_ms(infile):
"""
Summarize the contents of an MS directory in casacore table format
Parameters
----------
infile : str
input filename of MS
Returns
-------
pandas.core.frame.DataFrame
Summary information
"""
import os
import pandas as pd
import numpy as np
import cngi._utils._table_conversion2 as tblconv
from casacore import tables
infile = os.path.expanduser(infile) # does nothing if $HOME is unknown
if not infile.endswith('/'): infile = infile + '/'
# as part of MSv3 conversion, these columns in the main table are no longer needed
ignorecols = [
'FLAG_CATEGORY', 'FLAG_ROW', 'SIGMA', 'WEIGHT_SPECTRUM', 'DATA_DESC_ID'
]
# figure out characteristics of main table from select subtables (must all be present)
spw_xds = tblconv.read_simple_table(infile,
subtable='SPECTRAL_WINDOW',
ignore=ignorecols,
add_row_id=True)
pol_xds = tblconv.read_simple_table(infile,
subtable='POLARIZATION',
ignore=ignorecols)
ddi_xds = tblconv.read_simple_table(infile,
subtable='DATA_DESCRIPTION',
ignore=ignorecols)
ddis = list(ddi_xds['d0'].values)
summary = pd.DataFrame([])
spw_ids = ddi_xds.spectral_window_id.values
pol_ids = ddi_xds.polarization_id.values
chans = spw_xds.NUM_CHAN.values
pols = pol_xds.NUM_CORR.values
for ddi in ddis:
print('processing ddi %i of %i' % (ddi + 1, len(ddis)), end='\r')
sorted_table = tables.taql('select * from %s where DATA_DESC_ID = %i' %
(infile, ddi))
sdf = {
'ddi':
ddi,
'spw_id':
spw_ids[ddi],
'pol_id':
pol_ids[ddi],
'rows':
sorted_table.nrows(),
'times':
len(np.unique(sorted_table.getcol('TIME'))),
'baselines':
len(
np.unique(np.hstack([
sorted_table.getcol(rr)[:, None]
for rr in ['ANTENNA1', 'ANTENNA2']
]),
axis=0)),
'chans':
chans[spw_ids[ddi]],
'pols':
pols[pol_ids[ddi]]
}
sdf['size_MB'] = np.ceil(
(sdf['times'] * sdf['baselines'] * sdf['chans'] * sdf['pols'] * 9)
/ 1024**2).astype(int)
summary = pd.concat(
[summary, pd.DataFrame(sdf, index=[str(ddi)])], axis=0, sort=False)
sorted_table.close()
print(' ' * 50, end='\r')
return summary.set_index('ddi').sort_index()
def convert_expanded_table(infile,
outfile,
subsel=None,
ignore=None,
compressor=None,
chunks=(100, 100, 20, 2),
nofile=False):
if ignore is None: ignore = []
if compressor is None:
compressor = Blosc(cname='zstd', clevel=2, shuffle=0)
# select just the specified ddi
subseltql = '' if subsel is None else ' where DATA_DESC_ID = %s' % str(
subsel)
tb_tool = tables.taql('select * from %s%s' % (infile, subseltql))
# handle no chunking of first axis
chunks = [
chunks[di] if di < len(chunks) else chunks[-1] for di in range(4)
]
if nofile or (chunks[0] == -1):
chunks[0] = tb_tool.nrows()
# pointing table will use dims of time by antenna_id
# main table uses time x (antenna1,antenna2)
if infile.endswith('POINTING'):
baselines = np.unique(tb_tool.getcol('ANTENNA_ID', 0, -1))
mcoords = dict([('time', []),
('antenna_id',
xarray.DataArray(baselines, dims=['antenna_id']))])
else:
ant1, ant2 = tb_tool.getcol('ANTENNA1', 0,
-1), tb_tool.getcol('ANTENNA2', 0, -1)
baselines = np.array([
str(ll[0]).zfill(3) + '_' + str(ll[1]).zfill(3)
for ll in np.unique(np.hstack([ant1[:, None], ant2[:, None]]),
axis=0)
])
mcoords = dict([
('time', []),
('baseline',
xarray.DataArray(np.arange(len(baselines)).astype('int64'),
dims=['baseline']))
])
dims = [
'time', 'antenna_id', 'd2', 'd3'
] if infile.endswith('POINTING') else ['time', 'baseline', 'chan', 'pol']
cols = tb_tool.colnames()
times, mvars, mchunks, xds = [], {}, {}, xarray.Dataset()
ts_tb = tables.taql('select DISTINCT TIME from %s%s' % (infile, subseltql))
utimes = np.unique(ts_tb.getcol('TIME', 0, -1))
ts_tb.close()
subseltql = ' where ' if subsel is None else ' where DATA_DESC_ID = %s AND ' % str(
subsel)
# for each batch of times in a chunk
for ts in range(0, len(utimes), chunks[0]):
ts_tb = tables.taql('select * from %s%sTIME BETWEEN %f and %f' %
(infile, subseltql, utimes[ts], utimes[min(
len(utimes) - 1, ts + chunks[0] - 1)]))
raw_times = ts_tb.getcol('TIME', 0, -1)
times = np.unique(raw_times)
tidxs = np.searchsorted(utimes, raw_times) - ts
# compute the indices of all the times and baselines at this timestep
if infile.endswith('POINTING'):
ts_bases = ts_tb.getcol('ANTENNA_ID', 0, -1)
else:
ts_ant1, ts_ant2 = ts_tb.getcol('ANTENNA1', 0, -1), ts_tb.getcol(
'ANTENNA2', 0, -1)
ts_bases = np.array([
str(ll[0]).zfill(3) + '_' + str(ll[1]).zfill(3)
for ll in np.hstack([ts_ant1[:, None], ts_ant2[:, None]])
])
bidxs = np.searchsorted(baselines, ts_bases)
# extract data for each col
for col in cols:
if (col in ignore + ['TIME']) or (not ts_tb.iscelldefined(col, 0)):
continue
data = np.array(ts_tb.getcol(col, 0, -1))
if data.size == 0: continue
fulldata = np.full((
len(times),
len(baselines),
) + data.shape[1:],
np.nan,
dtype=data.dtype)
fulldata[tidxs, bidxs] = data
mchunks = dict(
zip(dims[:len(fulldata.shape)], [
chunks[ii] if chunks[ii] > 0 else fulldata.shape[ii]
for ii in range(len(fulldata.shape))
]))
mvars[col.upper()] = xarray.DataArray(
fulldata, dims=dims[:len(fulldata.shape)]).chunk(mchunks)
mcoords.update({
'time':
xarray.DataArray(convert_time(np.array(times)), dims=['time'])
})
xds = xarray.Dataset(mvars, coords=mcoords)
if (not nofile) and (ts == 0):
encoding = dict(
zip(list(xds.data_vars), cycle([{
'compressor': compressor
}])))
xds.to_zarr(outfile,
mode='w',
encoding=encoding,
consolidated=True)
elif not nofile:
xds.to_zarr(outfile,
mode='a',
append_dim='time',
compute=True,
consolidated=True)
print('processed %s time steps' % str(ts + 1), end='\r')
ts_tb.close()
if (not nofile) and (tb_tool.nrows() > 0):
xds = xarray.open_zarr(outfile)
tb_tool.close()
return xds
def get_flags(self):
"""
"""
ms = self.ms_file
return taql('select FLAG from $ms where ANTENNA1 != ANTENNA2')
def create_STATION_UVW_process(timeIndex, freqIndex, args):
"""
Inner-loop of create_STATION_UVW().
This function should be placed in create_STATION_UVW(), but due to limitations of joblib, it has been placed here.
:param timeIndex: time index
:param freqIndex: frequency index
:param args: output of parseArgs()
:return: tuple (timeIndex, freqIndex, STATION_INFO_UVW)
"""
import casacore.measures as cm
import casacore.quanta as cq
def ITRF_to_J2000(time, x, y, z):
dm = cm.measures()
dm.do_frame(dm.epoch('UTC', cq.quantity(time, 's')))
ITRF_position = dm.position(rf='ITRF',
v0=cq.quantity(x, 'm'),
v1=cq.quantity(y, 'm'),
v2=cq.quantity(z, 'm'))
dm.do_frame(ITRF_position)
ITRFLL_position = dm.measure(ITRF_position, 'ITRFLL')
height = ITRFLL_position['m2']
ITRFLL_direction = dm.direction('ITRFLL',
v0=ITRFLL_position['m0'],
v1=ITRFLL_position['m1'])
J2000_direction = dm.measure(ITRFLL_direction, 'J2000')
J2000_position = dm.position(rf='ITRF',
v0=J2000_direction['m0'],
v1=J2000_direction['m1'],
v2=height)
(az, el,
r) = (J2000_position['m0']['value'], J2000_position['m1']['value'],
J2000_position['m2']['value'])
return (az, el, r)
ITRF_to_J2000_vec = np.vectorize(ITRF_to_J2000)
store = openHDF5(args)
time = store['TIME_MAP'].loc[timeIndex].values
wavelength = sc.speed_of_light / store['FREQ_MAP'].loc[freqIndex].values
STATION_INFO_CART = store['STATION_INFO']
store.close()
(az, el, r) = ITRF_to_J2000_vec(time, STATION_INFO_CART['x'].values,
STATION_INFO_CART['y'].values,
STATION_INFO_CART['z'].values)
(x, y, z) = cc.sph2cart(az, el, r)
# pointing direction in radian (RA, DEC)
pointingDirection = ct.taql(
'select REFERENCE_DIR from {msFile}::FIELD'.format(
msFile=args['msFile'])).getcol('REFERENCE_DIR').squeeze()
M = cc.J2000_to_UVW_operator(*pointingDirection)
# TODO: different normalization for other datasets. Add a switch to control this.
(u, v, w) = M.dot(np.vstack((x, y, z))) / wavelength
STATION_INFO_UVW = pd.DataFrame({
'stationID': STATION_INFO_CART['stationID'],
'antennaID': STATION_INFO_CART['antennaID'],
'u': u,
'v': v,
'w': w
})
STATION_INFO_UVW = STATION_INFO_UVW[[
'stationID', 'antennaID', 'u', 'v', 'w'
]]
return timeIndex, freqIndex, STATION_INFO_UVW
'''
print '[MaSER] Forming baselines...'
ANTENNA1 = msfile.getcol('ANTENNA1')
ANTENNA2 = msfile.getcol('ANTENNA2')
ANTENNAS = set(ANTENNA1).union(set(ANTENNA2))
baselines = form_baselines(ANTENNAS)
print len(ANTENNA1)
print len(ANTENNA2)
print len(baselines)
print 'Formed %d baselines.' % len(baselines)
# Determine the number of correlations present.
polarizations = ct.taql('SELECT FROM %s::POLARIZATION' % (filename))
correlations = polarizations.getcol('NUM_CORR')
print 'Found %d correlations.' % correlations[0]
'''
Extract the visibilities per baseline. The measurement set is accessed using TaQL extracting
the columns into numpy arrays.
The final data is stored in regular text files, with each baseline and correlation having its
own file.
The columns will have the uvw coordinates, channel frequency, real and imaginary parts of the
visibilities and their corresponding errors.
'''
print '[MaSER] Extracting visibilities per baseline...'
try:
os.mkdir('visibilities')
except:
pass
die('DDE solutions not found!')
else:
LOGGER.info('Creating final 1'' image with DD solutions.')
CMD = 'DDF.py --Output-Name=image_dd --Data-MS={:s} --Deconv-PeakFactor 0.050000 --Data-ColName {ic:s} --Data-ChunkHours 4 --Parallel-NCPU=32 --Beam-CenterNorm=1 --Deconv-CycleFactor=0 --Deconv-MaxMinorIter=10000 --Deconv-MaxMajorIter=6 --Deconv-Mode SSD --Beam-Model=LOFAR --Beam-LOFARBeamMode=A --Weight-Mode Natural --Image-NPix=25000 --CF-wmax 50000 --CF-Nw 100 --Output-Also onNeds --Image-Cell {cell:f} --Facets-NFacets=7 --SSDClean-NEnlargeData 0 --Freq-NDegridBand 1 --Beam-NBand 1 --Facets-DiamMax 0.5 --Facets-DiamMin 0.1 --Deconv-RMSFactor=3.000000 --SSDClean-ConvFFTSwitch 10000 --Data-Sort 1 --Cache-Dir=. --Log-Memory 1 --GAClean-RMSFactorInitHMP 1.000000 --GAClean-MaxMinorIterInitHMP 10000.000000 --DDESolutions-SolsDir=SOLSDIR --Cache-Weight=reset --Output-Mode=Clean --Output-RestoringBeam {beam:s} --Weight-ColName="IMAGING_WEIGHT" --Freq-NBand=2 --RIME-DecorrMode=FT --SSDClean-SSDSolvePars [S,Alpha] --SSDClean-BICFactor 0 --Mask-Auto=1 --Mask-SigTh=10.00 --Selection-UVRangeKm=[5.0,2000.000000] --GAClean-MinSizeInit=10 --Mask-External=image_dirin_SSD_init_natural_m.app.restored.fits.mask.fits --DDESolutions-DDSols={ddsols:s}'.format(CONFIG['data']['mslist'], ic=CONFIG['image']['data_column'], cell=float(CONFIG['image']['cellsize_full'], beam=DDF_RESTORING_BEAM), ddsols=CONFIG['solutions']['ddsols_h5'])
LOGGER.info(CMD)
subprocess.call(CMD, shell=True)
LOGGER.info('Making PyBDSF catalogue of DD calibrated 1'' map.')
run_pybdsf(fitsname='image_dirin_SSD_init_natural_m.int.restored.fits', detectimage='image_dirin_SSD_init_natural_m.app.restored.fits', outcat='catalogue_1asec_final_dd')
IN_CATALGOUE = 'catalogue_1asec_final_dd.csv'
subprocess.call("sed '/^[[:space:]]*$/d' {:s} > {:s}".format(IN_CATALOGUE, 'catalogue_1asec_final_dd.sedded.csv'), shell=True)
if not CONFIG['mosaic'].getboolean('do_mosaic'):
LOGGER.info('Pipeline finished successfully.')
sys.exit(0)
LOGGER.info('Creating directions for full FoV 0.3'' imaging.')
tab = ct.taql('SELECT REFERENCE_DIR FROM {:s}::FIELD'.format(MSES[0]))
PHASECENTER = tab.getcol('REFERENCE_DIR').squeeze()
make_tiles(*PHASECENTER)
# Make phaseshifted copies of each tile and image.
DPPP_PARSETS = sorted(glob.glob('shift_to_facet_*.parset'))
for i, p in enumerate(DPPP_PARSETS):
for ms in MSES:
subprocess.call('DPPP {:s} msin={:s} msout={:s}'.format(p, ms, ms[:-3] + '.facet_{:02d}'.format(i), shell=True)
# Image here.
# WSClean with IDG on GPU, or DDFacet?