def apply_rfi_flagging (rfihist='RFIdata/rfipc.cp'):
"""Applies random flagging based on baseline length. Flagging percentages are taken
from the rfihist file"""
info("applying mock RFI flagging to $MS");
# import table of flagging percentages
(baselines,pc) = cPickle.load(file(rfihist));
info("max fraction is %.2f, min is %.2f"%(pc.max(),pc.min()));
# binsize is in km
binsize = baselines[0]*1000;
info("baseline binsize is %.2fm"%binsize);
# open MS, get the per-row baseline length,
tab = ms.msw();
uvw = tab.getcol('UVW')
uvw = numpy.sqrt((uvw**2).sum(1))
info("max baseline is %f"%uvw.max())
# convert it to uvbin
uvbin = numpy.int64(uvw/binsize)
info("max uv-bin is %d, we have %d bins tabulated"%(uvbin.max(),len(pc)));
uvbin[uvbin>=len(pc)] = len(pc)-1
# uvbin gives the number of the baseline bin in the 'pc' table above
# now pull out a random number [0,1} for each row
rr = numpy.random.random_sample(len(uvw))
## compute per-row threshold by looking up 'pc' array using the uvbin
thr = pc[uvbin]
## kill the unlucky rows
unlucky = rr<thr
info(">>> mock RFI applied, flagged fraction is %.2f"%(float(unlucky.sum())/len(thr)));
tab.putcol("FLAG_ROW",unlucky);
def simsky(msname='$MS',
lsmname='$LSM',
tdlsec='$TDLSEC',
tdlconf='$TDLCONF',
column='$COLUMN',
noise=0,
args=[],
addToCol=None,
**kw):
""" Simulates visibilities into an MS """
msname, lsmname, column, tdlsec, tdlconf = interpolate_locals(
'msname lsmname'
' column tdlsec tdlconf')
fits = True if verify_sky(lsmname) is 'FITS' else False
v.MS = msname
v.LSM = lsmname
_column = 'MODEL_DATA' if addToCol else column
if fits:
_column = 'MODEL_DATA' if noise else column
im.argo.predict_vis(lsmname, wprojplanes=128, column=_column)
if noise:
simnoise(noise=noise, addToCol=_column, column=column)
else:
args = ["${ms.MS_TDL} ${lsm.LSM_TDL}"] + list(args)
options = {}
options['ms_sel.output_column'] = _column
if noise:
options['noise_stddev'] = noise
mqt.run(TURBO_SIM,
job='_tdl_job_1_simulate_MS',
config=tdlconf,
section=tdlsec,
options=options,
args=args,
**kw)
if addToCol:
tab = ms.msw()
col1 = tab.getcol(addToCol)
col2 = tab.getcol('MODEL_DATA')
comb = col1 + col2
nrows = len(comb)
rowchunk = nrows // 5
for row0 in range(0, nrows, rowchunk):
nr = min(nrows - row0, rowchunk)
info('MODEL_DATA + $addToCol --> $column : rows %d-%d' %
(row0, row0 + nr))
tab.putcol(column, comb[row0:row0 + nr], row0, nr)
tab.close()
def swapfields (f1,f2):
"""Swaps two fields in an MS"""
info("swapping FIELDs $f1 and $f2 in $MS");
field = ms.msw(subtable="FIELD");
for name in field.colnames():
info("swapping column $name");
col = field.getcol(name);
col[f1],col[f2] = col[f2],col[f1];
field.putcol(name,col);
field.close();
tab = ms.msw();
fcol = tab.getcol("FIELD_ID");
r1 = (fcol==f1)
r2 = (fcol==f2)
fcol[r1] = f2
fcol[r2] = f1
tab.putcol("FIELD_ID",fcol);
tab.close();
def swapfields(f1, f2):
"""Swaps two fields in an MS"""
info("swapping FIELDs $f1 and $f2 in $MS")
field = ms.msw(subtable="FIELD")
for name in field.colnames():
info("swapping column $name")
col = field.getcol(name)
col[f1], col[f2] = col[f2], col[f1]
field.putcol(name, col)
field.close()
tab = ms.msw()
fcol = tab.getcol("FIELD_ID")
r1 = (fcol == f1)
r2 = (fcol == f2)
fcol[r1] = f2
fcol[r2] = f1
tab.putcol("FIELD_ID", fcol)
tab.close()
def fix_antpos ():
anttab = ms.msw(subtable="ANTENNA");
pos = anttab.getcol("POSITION");
wh = pos[:,1]>0;
if wh.any():
info(wh.sum(),"VLA dishes appear to be located in India. Moving them back to NM.")
pos[wh,1] = -pos[wh,1];
anttab.putcol("POSITION",pos);
else:
info("$MS antenna positions seem right, nothing to do")
def flag_stepped_timeslot (step=3):
"""Flags every Nth timeslot"""
nant = ms.ms(subtable="ANTENNA").nrows();
tab = ms.msw();
nb = nant*(nant+1)/2
frow = tab.getcol("FLAG_ROW");
nr = len(frow);
nt = len(frow)/nb;
info("$MS has $nr rows, $nant antennas, $nb baselines and $nt timeslots, flagging every $step timeslots");
frow = frow.reshape([nt,nb]);
frow[::step,:] = True;
tab.putcol("FLAG_ROW",frow.reshape((nr,)));
def fix_antpos():
anttab = ms.msw(subtable="ANTENNA")
pos = anttab.getcol("POSITION")
wh = pos[:, 1] > 0
if wh.any():
info(
wh.sum(),
"VLA dishes appear to be located in India. Moving them back to NM."
)
pos[wh, 1] = -pos[wh, 1]
anttab.putcol("POSITION", pos)
else:
info("$MS antenna positions seem right, nothing to do")
def addnoise (noise=0,rowchunk=100000):
"""adds noise to MODEL_DATA, writes to CORRECTED_DATA""";
# compute expected noise
noise = compute_vis_noise(noise);
# fill MS with noise
tab = ms.msw()
nrows = tab.nrows();
for row0 in range(0,nrows,rowchunk):
nr = min(rowchunk,nrows-row0);
info("Copying MODEL_DATA+noise to CORRECTED_DATA (rows $row0 to %d)"%(row0+nr-1));
data = tab.getcol("MODEL_DATA",row0,nr);
data += noise*(numpy.random.randn(*data.shape) + 1j*numpy.random.randn(*data.shape));
tab.putcol("CORRECTED_DATA",data,row0,nr)
tab.close()
def simsky(msname='$MS', lsmname='$LSM', tdlsec='$TDLSEC', tdlconf='$TDLCONF',
column='$COLUMN', noise=0, args=[],
addToCol=None,**kw):
""" Simulates visibilities into an MS """
msname, lsmname, column, tdlsec, tdlconf = interpolate_locals('msname lsmname'
' column tdlsec tdlconf')
fits = True if verify_sky(lsmname) is 'FITS' else False
v.MS = msname
v.LSM = lsmname
_column = 'MODEL_DATA' if addToCol else column
if fits:
_column = 'MODEL_DATA' if noise else column
im.argo.predict_vis(lsmname, wprojplanes=128, column=_column)
if noise:
simnoise(noise=noise,addToCol=_column,column=column)
else:
args = ["${ms.MS_TDL} ${lsm.LSM_TDL}"] + list(args)
options = {}
options['ms_sel.output_column'] = _column
if noise:
options['noise_stddev'] = noise
mqt.run(TURBO_SIM, job='_tdl_job_1_simulate_MS',
config=tdlconf, section=tdlsec, options=options, args=args,**kw)
if addToCol:
tab = ms.msw()
col1 = tab.getcol(addToCol)
col2 = tab.getcol('MODEL_DATA')
comb = col1 + col2
nrows = len(comb)
rowchunk = nrows//5
for row0 in range(0,nrows,rowchunk):
nr = min(nrows-row0,rowchunk)
info('MODEL_DATA + $addToCol --> $column : rows %d-%d'%(row0,row0+nr) )
tab.putcol(column,comb[row0:row0+nr],row0,nr)
tab.close()
def simnoise (noise=0,rowchunk=100000,skipnoise=False,addToCol=None,scale_noise=1.0,column='MODEL_DATA'):
conf = MS.split('_')[0]
spwtab = ms.ms(subtable="SPECTRAL_WINDOW")
freq0 = spwtab.getcol("CHAN_FREQ")[ms.SPWID,0]/1e6
tab = ms.msw()
dshape = list(tab.getcol('DATA').shape)
nrows = dshape[0]
noise = noise or compute_vis_noise()
if addToCol: colData = tab.getcol(addToCol)
for row0 in range(0,nrows,rowchunk):
nr = min(rowchunk,nrows-row0)
dshape[0] = nr
data = noise*(numpy.random.randn(*dshape) + 1j*numpy.random.randn(*dshape)) * scale_noise
if addToCol:
data+=colData[row0:(row0+nr)]
info(" $addToCol + noise --> CORRECTED_DATA (rows $row0 to %d)"%(row0+nr-1))
column = 'CORRECTED_DATA'
else : info("Adding noise to $column (rows $row0 to %d)"%(row0+nr-1))
tab.putcol(column,data,row0,nr);
tab.close()
def clear_flags ():
tab = ms.msw();
tab.putcol("FLAG_ROW",numpy.zeros(tab.nrows(),bool));
info("cleared all flags in $MS");