def everythings_flagcali(msname='', spw='8~12'):
# Narrow-time, broad-band RFI flag at both calibrators at non-continuum
# Extend flag is broken. So separate this to a two-stage flagging
everythings_log("Extra flagging for calibrators")
flagdata(vis=msname,
intent='*CALI*',
mode='tfcrop',
datacolumn='corrected',
spw=spw,
extendflags=False,
action='apply',
display='',
freqcutoff=3.0,
timecutoff=2.0,
combinescans=True,
winsize=1,
flagdimension='timefreq')
flagdata(
vis=msname,
intent='*CALI*',
mode='extend',
datacolumn='corrected',
spw=spw,
extendflags=True,
action='apply',
display='',
freqcutoff=100.0,
timecutoff=100.0,
growtime=70.0,
growfreq=40.0, # weak time due to quack
extendpols=True,
combinescans=False,
winsize=1,
flagdimension='timefreq')
def test_caltable_flagbackup(self):
'''Flagmanager:: cal table mode=list, flagbackup=True/False'''
# Need a fresh start
os.system('rm -rf cal.fewscans.bpass*')
self.setUp_bpass_case()
self.unflag_ms()
flagmanager(vis=self.vis, mode='list')
aflocal.open(self.vis)
self.assertEqual(len(aflocal.getflagversionlist()), 2)
aflocal.done()
flagdata(vis=self.vis, mode='unflag', flagbackup=False)
flagmanager(vis=self.vis, mode='list')
aflocal.open(self.vis)
self.assertEqual(len(aflocal.getflagversionlist()), 2)
aflocal.done()
flagdata(vis=self.vis, mode='unflag', flagbackup=True)
flagmanager(vis=self.vis, mode='list')
aflocal.open(self.vis)
self.assertEqual(len(aflocal.getflagversionlist()), 3)
aflocal.done()
newname = 'Ha! The best version ever!'
flagmanager(vis=self.vis, mode='rename', oldname='flagdata_1', versionname=newname,
comment='This is a *much* better name')
flagmanager(vis=self.vis, mode='list')
aflocal.open(self.vis)
self.assertEqual(len(aflocal.getflagversionlist()), 3)
aflocal.done()
self.assertTrue(os.path.exists(self.vis+'.flagversions/flags.'+newname),
'Flagversion file does not exist: flags.'+newname)
def test_save_cal(self):
'''Flagcmd: list flag cmds from MS and save to a file'''
self.setUp_data4rflag()
msfile = self.vis
flagcmd(vis=msfile, action='clear', clearall=True)
# Save cmds to FLAG_CMD table
flagcmd(vis=msfile, inpmode='list', inpfile=["spw='1,3",
"mode='clip' clipminmax=[0,500] datacolumn='SNR'"],
action='list', savepars=True)
self.setUp_bpass_case()
flagcmds = 'calflags.txt'
if os.path.exists(flagcmds):
os.system('rm -rf '+flagcmds)
# Apply to cal table and save to an external file
flagcmd(vis=self.vis, inpmode='table', inpfile=msfile, savepars=True, outfile=flagcmds)
self.assertTrue(os.path.exists(flagcmds))
res = flagdata(vis=self.vis, mode='summary')
self.assertEqual(res['flagged'], 246315)
# Apply from list in file and compare
flagdata(vis=self.vis, mode='unflag')
flagcmd(vis=self.vis, inpmode='list', inpfile=flagcmds, action='apply')
res = flagdata(vis=self.vis, mode='summary')
self.assertEqual(res['flagged'], 246315)
def test_mpi4casa_flagdata_list_return(self):
"""Test support for MMS using flagdata in unflag+clip mode"""
# Create list file
text = "mode='unflag'\n"\
"mode='clip' clipminmax=[0,0.1]"
filename = 'list_flagdata.txt'
create_input(text, filename)
# step 1: Do unflag+clip
flagdata(vis=self.vis, mode='list', inpfile=filename)
# step 2: Now do summary
ret_dict = flagdata(vis=self.vis, mode='summary')
# Check summary
self.assertTrue(ret_dict['name']=='Summary')
self.assertTrue(ret_dict['spw']['15']['flagged'] == 96284.0)
self.assertTrue(ret_dict['spw']['0']['flagged'] == 129711.0)
self.assertTrue(ret_dict['spw']['1']['flagged'] == 128551.0)
self.assertTrue(ret_dict['spw']['2']['flagged'] == 125686.0)
self.assertTrue(ret_dict['spw']['3']['flagged'] == 122862.0)
self.assertTrue(ret_dict['spw']['4']['flagged'] == 109317.0)
self.assertTrue(ret_dict['spw']['5']['flagged'] == 24481.0)
self.assertTrue(ret_dict['spw']['6']['flagged'] == 0)
self.assertTrue(ret_dict['spw']['7']['flagged'] == 0)
self.assertTrue(ret_dict['spw']['8']['flagged'] == 0)
self.assertTrue(ret_dict['spw']['9']['flagged'] == 27422.0)
self.assertTrue(ret_dict['spw']['10']['flagged'] == 124638.0)
self.assertTrue(ret_dict['spw']['11']['flagged'] == 137813.0)
self.assertTrue(ret_dict['spw']['12']['flagged'] == 131896.0)
self.assertTrue(ret_dict['spw']['13']['flagged'] == 125074.0)
self.assertTrue(ret_dict['spw']['14']['flagged'] == 118039.0)
def everythings_mwhi(msname='',
spw_HI=8,
restfreq=1420405752,
numChanPhase=200,
numChanBandpass=100):
# spw_HI: int
# restfreq: int. in Hz.
# numChan: int
msmd.open(msname)
freqs = msmd.chanfreqs(spw_HI)
freqs_delta = np.mean(freqs[1:] - freqs[:-1])
# i=0: Phase calibrator
# i=1: Bandpass calibrator:
for i in range(2):
numChan = numChanPhase if i == 0 else numChanBandpass
min_freq = restfreq - freqs_delta * numChan
max_freq = restfreq + freqs_delta * numChan
mask = (freqs < max_freq) * (freqs > min_freq)
if np.sum(mask) > 0:
temp = np.arange(len(mask), dtype=int)
chan_ids = temp[mask]
min_chan, max_chan = \
str(np.min(chan_ids)), str(np.max(chan_ids) + 1)
rm_spw = str(spw_HI) + ':' + min_chan + '~' + max_chan
intent = '*PHASE*' if i == 0 else '*BANDPASS*'
everythings_log("Removing MW HI in " + intent + " calibrator.")
flagdata(vis=msname, spw=rm_spw, intent=intent)
del rm_spw, chan_ids, temp, min_chan, max_chan
del mask, max_freq, min_freq
msmd.close()
def test_uquack(self):
'''flagcmd: unapply quack agent'''
# Remove any cmd from table
flagcmd(vis=self.vis, action='clear', clearall=True)
# Flag using the quack agent
myinput = "scan='1~3' mode=quack quackinterval=1.0"
filename = create_input(myinput)
flagcmd(vis=self.vis, inpmode='list', inpfile=filename, action='apply', savepars=True,
flagbackup=False)
result = flagdata(vis=self.vis,mode='summary')
quack_flags = result['scan']['1']['flagged']
# Flag using manual agent
myinput = "scan='1'"
filename = create_input(myinput)
flagcmd(vis=self.vis, inpmode='list', inpfile=filename, action='apply', savepars=True,
flagbackup=False)
result = flagdata(vis=self.vis,mode='summary')
scan1_flags = result['scan']['1']['flagged']
# Unapply only the quack line
flagcmd(vis=self.vis, action='unapply', useapplied=True, tablerows=0, savepars=True)
result = flagdata(vis=self.vis,mode='summary')
manual_flags = result['scan']['1']['flagged']
# Only the manual flags should be there
# self.assertEqual(result['flagged'], 568134, 'Expected 568134 flags, found %s'%result['flagged'])
# self.assertEqual(result['total'], 568134,'Expected total 568134, found %s'%result['total'])
# CAS-5377. New unapply action
self.assertEqual(result['scan']['3']['flagged'], 0)
self.assertEqual(manual_flags,scan1_flags - quack_flags)
def test_savepars(self):
'''importevla: save the flag commands and do not apply'''
msname = 'notapplied.ms'
cmdfile = msname.replace('.ms','_cmd.txt')
if os.path.exists(msname):
os.system('rm -rf '+msname)
if os.path.exists(cmdfile):
os.system('rm -rf '+cmdfile)
importevla(asdm=self.asdm, vis=msname,scans='11~13',online=True,flagzero=True,shadow=True,savecmds=True,
applyflags=False,flagbackup=False)
# Check flags
res = flagdata(vis=msname, mode='summary')
self.assertEqual(res['flagged'],0,'No flags should have been applied')
# Check output file existence
self.assertTrue(os.path.exists(cmdfile))
# Check file content
ff = open(cmdfile,'r')
cmdlist = ff.readlines()
ncmds = cmdlist.__len__()
ff.close()
self.assertEqual(ncmds, 216, 'Online, shadow and clip zeros should be saved to file')
# Apply flags using flagdata
flagdata(vis=msname, mode='list', inpfile=cmdfile)
res = flagdata(vis=msname, mode='summary')
self.assertEqual(res['flagged'],6090624)
def test_utfcrop(self):
'''flagcmd: unapply tfcrop agent'''
# Remove any cmd from table
flagcmd(vis=self.vis, action='clear', clearall=True)
# Flag using manual agent
myinput = "scan='1'"
filename = create_input(myinput)
flagcmd(vis=self.vis, inpmode='list', inpfile=filename, action='apply', savepars=True)
# Flag using tfcrop agent from file
# Note : For this test, scan=4 gives identical flags on 32/64 bit machines,
# and one flag difference on a Mac (32)
# Other scans give differences at the 0.005% level.
myinput = "scan='4' mode=tfcrop correlation='ABS_RR' extendflags=False"
filename = create_input(myinput)
flagcmd(vis=self.vis, inpmode='list', inpfile=filename, action='apply', savepars=True,
flagbackup=False)
res = flagdata(vis=self.vis,mode='summary')
self.assertEqual(res['scan']['1']['flagged'], 568134, 'Whole scan=1 should be flagged')
#self.assertEqual(res['scan']['4']['flagged'], 1201, 'scan=4 should be partially flagged')
self.assertTrue(res['scan']['4']['flagged']>= 1200 and res['scan']['4']['flagged']<= 1204, \
'scan=4 should be partially flagged')
# Unapply only the tfcrop line
flagcmd(vis=self.vis, action='unapply', useapplied=True, tablerows=1, savepars=False)
result = flagdata(vis=self.vis,mode='summary',scan='4')
self.assertEqual(result['flagged'], 0, 'Expected 0 flags, found %s'%result['flagged'])
self.assertEqual(result['total'], 95256,'Expected total 95256, found %s'%result['total'])
def test_savepars(self):
'''importevla: save the flag commands and do not apply'''
msname = 'notapplied.ms'
cmdfile = msname.replace('.ms','_cmd.txt')
if os.path.exists(msname):
os.system('rm -rf '+msname)
if os.path.exists(cmdfile):
os.system('rm -rf '+cmdfile)
importevla(asdm=self.asdm, vis=msname,scans='11~13',online=True,flagzero=True,shadow=True,savecmds=True,
applyflags=False,flagbackup=False)
# Check flags
res = flagdata(vis=msname, mode='summary')
self.assertEqual(res['flagged'],0,'No flags should have been applied')
# Check output file existence
self.assertTrue(os.path.exists(cmdfile))
# Check file content
ff = open(cmdfile,'r')
cmdlist = ff.readlines()
ncmds = cmdlist.__len__()
ff.close()
self.assertEqual(ncmds, 216, 'Online, shadow and clip zeros should be saved to file')
# Apply flags using flagdata
flagdata(vis=msname, mode='list', inpfile=cmdfile)
res = flagdata(vis=msname, mode='summary')
self.assertEqual(res['flagged'],6090624)
def test_flagversions(self):
'''Partition: check that the .flagversions is created'''
# Run partition and create the .flagversions
partition(vis=self.msfile,
outputvis=self.mmsfile,
createmms=True,
disableparallel=True,
datacolumn='data')
self.assertTrue(os.path.exists(self.mmsfile + '.flagversions'))
# Check that the number of backups in MMS is correct
aflocal = aftool()
aflocal.open(self.mmsfile)
nv = aflocal.getflagversionlist()
aflocal.done()
self.assertEqual(len(nv), 3)
# Run flagdata on MMS to check if it works well.
flagdata(vis=self.mmsfile, mode='unflag', flagbackup=True)
# Check that the number of backups in MMS is correct
aflocal = aftool()
aflocal.open(self.mmsfile)
nvref = aflocal.getflagversionlist()
aflocal.done()
self.assertEqual(len(nvref), 4)
def test2m(self):
"""flagmanager test2m: Create, then restore autoflag"""
self.unflag_ms()
flagdata(vis=self.vis, mode='summary')
flagmanager(vis=self.vis)
flagdata(vis=self.vis, mode='manual', antenna="2", flagbackup=True)
flagmanager(vis=self.vis)
ant2 = flagdata(vis=self.vis, mode='summary')['flagged']
print "After flagging antenna 2 there were", ant2, "flags"
# Change flags, then restore
flagdata(vis=self.vis, mode='manual', antenna="3", flagbackup=True)
flagmanager(vis = self.vis)
ant3 = flagdata(vis=self.vis, mode='summary')['flagged']
print "After flagging antenna 2 and 3 there were", ant3, "flags"
flagmanager(vis=self.vis, mode='restore', versionname='flagdata_2')
restore2 = flagdata(vis=self.vis, mode='summary')['flagged']
print "After restoring pre-antenna 2 flagging, there are", restore2, "flags; should be", ant2
self.assertEqual(restore2, ant2)
def test2m(self):
"""flagmanager test2m: Create, then restore autoflag"""
self.unflag_ms()
flagdata(vis=self.vis, mode='summary')
flagmanager(vis=self.vis)
flagdata(vis=self.vis, mode='manual', antenna="2", flagbackup=True)
flagmanager(vis=self.vis)
ant2 = flagdata(vis=self.vis, mode='summary')['flagged']
print "After flagging antenna 2 there were", ant2, "flags"
# Change flags, then restore
flagdata(vis=self.vis, mode='manual', antenna="3", flagbackup=True)
flagmanager(vis=self.vis)
ant3 = flagdata(vis=self.vis, mode='summary')['flagged']
print "After flagging antenna 2 and 3 there were", ant3, "flags"
flagmanager(vis=self.vis, mode='restore', versionname='flagdata_2')
restore2 = flagdata(vis=self.vis, mode='summary')['flagged']
print "After restoring pre-antenna 2 flagging, there are", restore2, "flags; should be", ant2
self.assertEqual(restore2, ant2)
def test_apply4(self):
'''importevla: Save online flags to FLAG_CMD and file; do not apply'''
# Use default msname and outfile
msname = 'xosro4.ms'
cmdfile = msname.replace('.ms', '_cmd.txt')
if os.path.exists(msname):
os.system('rm -rf ' + msname)
if os.path.exists(cmdfile):
os.system('rm -rf ' + cmdfile)
importevla(asdm=self.asdm,
vis=msname,
scans='2',
online=True,
shadow=False,
flagzero=False,
applyflags=False,
savecmds=True,
flagbackup=False)
# No flags were applied
res = flagdata(vis=msname, mode='summary')
self.assertEqual(res['flagged'], 0)
# Apply only row 213 using flagcmd
# The command in row 213 is the following:
# antenna='ea06' timerange='2012/02/22/22:30:55.200~2012/02/22/22:35:08.199'
# spw='EVLA_X#A0C0#0' correlation='LL,LR,RL
flagcmd(vis=msname, action='apply', tablerows=213)
# Check flags. RR should no be flagged
res = flagdata(vis=msname, mode='summary')
self.assertEqual(res['correlation']['RR']['flagged'], 0,
'RR should not be flagged')
self.assertEqual(res['correlation']['LL']['flagged'], 29440)
self.assertEqual(res['correlation']['LR']['flagged'], 29440)
self.assertEqual(res['correlation']['RL']['flagged'], 29440)
self.assertEqual(res['antenna']['ea06']['flagged'], 88320)
self.assertEqual(res['antenna']['ea07']['flagged'], 3840,
'Only a few baselines should be flagged')
self.assertEqual(res['antenna']['ea08']['flagged'], 3840,
'Only a few baselines should be flagged')
# Check output file existence
self.assertTrue(os.path.exists(cmdfile))
# Check file content
ff = open(cmdfile, 'r')
cmdlist = ff.readlines()
ncmds = cmdlist.__len__()
ff.close()
self.assertEqual(ncmds, 214,
'Only Online cmds should have been saved to file')
def test_MS_flagcmds(self):
"""Flagcmd:: Save flags to MS and apply to cal table"""
self.setUp_data4rflag()
msfile = self.vis
flagcmd(vis=msfile, action='clear', clearall=True)
flagdata(vis=msfile, antenna='ea09', action='', savepars=True)
self.setUp_bpass_case()
flagcmd(vis=self.vis, inpfile=msfile, action='apply', flagbackup=False)
summary=flagdata(vis=self.vis, mode='summary')
self.assertEqual(summary['antenna']['ea09']['flagged'], 48000.0)
self.assertEqual(summary['antenna']['ea10']['flagged'], 0.0)
def test_apply4(self):
"""importevla: Save online flags to FLAG_CMD and file; do not apply"""
# Use default msname and outfile
msname = "xosro4.ms"
cmdfile = msname.replace(".ms", "_cmd.txt")
if os.path.exists(msname):
os.system("rm -rf " + msname)
if os.path.exists(cmdfile):
os.system("rm -rf " + cmdfile)
importevla(
asdm=self.asdm,
vis=msname,
scans="2",
online=True,
shadow=False,
flagzero=False,
applyflags=False,
savecmds=True,
flagbackup=False,
)
# No flags were applied
res = flagdata(vis=msname, mode="summary")
self.assertEqual(res["flagged"], 0)
# Apply only row 213 using flagcmd
# The command in row 213 is the following:
# antenna='ea06' timerange='2012/02/22/22:30:55.200~2012/02/22/22:35:08.199'
# spw='EVLA_X#A0C0#0' correlation='LL,LR,RL
flagcmd(vis=msname, action="apply", tablerows=213)
# Check flags. RR should no be flagged
res = flagdata(vis=msname, mode="summary")
self.assertEqual(res["correlation"]["RR"]["flagged"], 0, "RR should not be flagged")
self.assertEqual(res["correlation"]["LL"]["flagged"], 29440)
self.assertEqual(res["correlation"]["LR"]["flagged"], 29440)
self.assertEqual(res["correlation"]["RL"]["flagged"], 29440)
self.assertEqual(res["antenna"]["ea06"]["flagged"], 88320)
self.assertEqual(res["antenna"]["ea07"]["flagged"], 3840, "Only a few baselines should be flagged")
self.assertEqual(res["antenna"]["ea08"]["flagged"], 3840, "Only a few baselines should be flagged")
# Check output file existence
self.assertTrue(os.path.exists(cmdfile))
# Check file content
ff = open(cmdfile, "r")
cmdlist = ff.readlines()
ncmds = cmdlist.__len__()
ff.close()
self.assertEqual(ncmds, 214, "Only Online cmds should have been saved to file")
def test_dictionary1(self):
'''flagmanager: Test the returned dictionary in list mode'''
self.unflag_ms()
# MS should start only with the standard info such as:
# MS : /diska/scastro/work/cas11462/nosedir/cal.fewscans.bpass
# main : working copy in main table
aflocal.open(self.vis)
self.assertEqual(len(aflocal.getflagversionlist()), 2)
aflocal.done()
flagsdict1 = flagmanager(vis=self.vis, mode='list')
self.assertIsInstance(
flagsdict1, dict,
'mode=list should return a dictionary of the flag versions')
flagdata(vis=self.vis, mode='unflag', flagbackup=False)
flagsdict2 = flagmanager(vis=self.vis, mode='list')
self.assertDictEqual(
flagsdict1, flagsdict2,
'Dictionaries should contain the same flag versions')
flagdata(vis=self.vis, mode='unflag', flagbackup=True)
flagsdict3 = flagmanager(vis=self.vis, mode='list')
self.assertEqual(flagsdict3[0]['name'], 'flagdata_1')
# Rename the flagdata_1 version
newname = 'A new name'
flagmanager(vis=self.vis,
mode='rename',
oldname='flagdata_1',
versionname=newname,
comment='A new versions name')
flagsdict4 = flagmanager(vis=self.vis, mode='list')
self.assertTrue(
os.path.exists(self.vis + '.flagversions/flags.' + newname),
'Flagversion file does not exist: flags.' + newname)
self.assertEqual(flagsdict4[0]['name'], newname,
'Dictionary should show new name')
# Specific for MMS
if testmms:
areg = self.vis + '/SUBMSS/*flagversions*'
import glob
print 'Check for .flagversions in the wrong place.'
self.assertEqual(
glob.glob(areg), [],
'There should not be any .flagversions in the'
' SUBMSS directory')
def test_flagsrestore(self):
"""Partition: check that we can restore the flags"""
# Delete any flagversions
if os.path.exists(self.msfile + ".flagversions"):
shutil.rmtree(self.msfile + ".flagversions")
# Unflag the MS
flagdata(vis=self.msfile, mode="unflag", flagbackup=False)
# Run partition and create the .flagversions
partition(vis=self.msfile, outputvis=self.mmsfile, createmms=True, disableparallel=True, datacolumn="data")
self.assertTrue(os.path.exists(self.mmsfile + ".flagversions"))
# Flag spw=9 and then spw=7 in the MMS
flagdata(vis=self.mmsfile, mode="manual", spw="9", flagbackup=True)
flagdata(vis=self.mmsfile, mode="manual", spw="7", flagbackup=True)
# There should be flags in spw=7 and 9 in MMS
res = flagdata(vis=self.mmsfile, mode="summary")
self.assertEqual(res["flagged"], 549888)
# Restore the original flags (there should be none)
flagmanager(vis=self.mmsfile, mode="restore", versionname="partition_1")
res = flagdata(vis=self.mmsfile, mode="summary")
self.assertEqual(res["flagged"], 0)
def test_flagsrestore(self):
'''Partition: check that we can restore the flags'''
# Delete any flagversions
if os.path.exists(self.msfile+'.flagversions'):
shutil.rmtree(self.msfile+'.flagversions')
# Unflag the MS
flagdata(vis=self.msfile, mode='unflag', flagbackup=False)
# Run partition and create the .flagversions
partition(vis=self.msfile, outputvis=self.mmsfile, createmms=True,
disableparallel=True, datacolumn='data')
self.assertTrue(os.path.exists(self.mmsfile+'.flagversions'))
# Flag spw=9 and then spw=7 in the MMS
flagdata(vis=self.mmsfile, mode='manual', spw='9', flagbackup=True)
flagdata(vis=self.mmsfile, mode='manual', spw='7', flagbackup=True)
# There should be flags in spw=7 and 9 in MMS
res = flagdata(vis=self.mmsfile, mode='summary')
self.assertEqual(res['flagged'],549888)
# Restore the original flags (there should be none)
flagmanager(vis=self.mmsfile, mode='restore', versionname='partition_1')
res = flagdata(vis=self.mmsfile, mode='summary')
self.assertEqual(res['flagged'],0)
def test_flagsrestore(self):
'''Partition: check that we can restore the flags'''
# Delete any flagversions
if os.path.exists(self.msfile + '.flagversions'):
shutil.rmtree(self.msfile + '.flagversions')
# Unflag the MS
flagdata(vis=self.msfile, mode='unflag', flagbackup=False)
# Run partition and create the .flagversions
partition(vis=self.msfile,
outputvis=self.mmsfile,
createmms=True,
disableparallel=True,
datacolumn='data')
self.assertTrue(os.path.exists(self.mmsfile + '.flagversions'))
# Flag spw=9 and then spw=7 in the MMS
flagdata(vis=self.mmsfile, mode='manual', spw='9', flagbackup=True)
flagdata(vis=self.mmsfile, mode='manual', spw='7', flagbackup=True)
# There should be flags in spw=7 and 9 in MMS
res = flagdata(vis=self.mmsfile, mode='summary')
self.assertEqual(res['flagged'], 549888)
# Restore the original flags (there should be none)
flagmanager(vis=self.mmsfile,
mode='restore',
versionname='partition_1')
res = flagdata(vis=self.mmsfile, mode='summary')
self.assertEqual(res['flagged'], 0)
def test_default_cparam(self):
'''Flagcmd: flag CPARAM data column'''
flist = ["mode='clip' clipzeros=True datacolumn='CPARAM'"]
flagcmd(vis=self.vis, inpmode='list', inpfile=flist,
flagbackup=False)
res = flagdata(vis=self.vis, mode='summary')
self.assertEqual(res['flagged'], 11078, 'Should use CPARAM as the default column')
def test_unsupported_mode_in_list(self):
'''Flagcmd: elevation and shadow are not supported in cal tables'''
res = flagcmd(vis=self.vis, inpmode='list', inpfile=["mode='elevation'",
"spw='1'"])
res = flagdata(vis=self.vis, mode='summary')
self.assertEqual(res['spw']['1']['flagged'], 83200)
def test_apply3(self):
'''importevla: apply clip zeros on RR and LL and save to file'''
msname = 'zeros.ms'
cmdfile = 'zeros_cmd.txt'
if os.path.exists(msname):
os.system('rm -rf '+msname)
if os.path.exists(cmdfile):
os.system('rm -rf '+cmdfile)
importevla(asdm=self.asdm, vis=msname, scans='2,13',online=False, applyflags=True,
shadow=False,flagzero=True,flagpol=False, flagbackup=False, savecmds=True)
# Check flags
res = flagdata(vis=msname, mode='summary')
self.assertEqual(res['flagged'],0,'There are no zeros in this data set')
self.assertEqual(res['scan']['2']['flagged'],0,'No flags should have been applied')
self.assertEqual(res['scan']['13']['flagged'],0,'No flags should have been applied')
# Check output file existence
self.assertTrue(os.path.exists(cmdfile))
# Check output file existence
self.assertTrue(os.path.exists(cmdfile))
# Check file content
ff = open(cmdfile,'r')
cmdlist = ff.readlines()
ncmds = cmdlist.__len__()
self.assertEqual(ncmds, 2, 'Only clip zeros should be saved to file')
def test_apply3(self):
'''importevla: apply clip zeros on RR and LL and save to file'''
msname = 'zeros.ms'
cmdfile = 'zeros_cmd.txt'
if os.path.exists(msname):
os.system('rm -rf '+msname)
if os.path.exists(cmdfile):
os.system('rm -rf '+cmdfile)
importevla(asdm=self.asdm, vis=msname, scans='2,13',online=False, applyflags=True,
shadow=False,flagzero=True,flagpol=False, flagbackup=False, savecmds=True)
# Check flags
res = flagdata(vis=msname, mode='summary')
self.assertEqual(res['flagged'],0,'There are no zeros in this data set')
self.assertEqual(res['scan']['2']['flagged'],0,'No flags should have been applied')
self.assertEqual(res['scan']['13']['flagged'],0,'No flags should have been applied')
# Check output file existence
self.assertTrue(os.path.exists(cmdfile))
# Check output file existence
self.assertTrue(os.path.exists(cmdfile))
# Check file content
ff = open(cmdfile,'r')
cmdlist = ff.readlines()
ncmds = cmdlist.__len__()
self.assertEqual(ncmds, 2, 'Only clip zeros should be saved to file')
def test_clip_one_list(self):
'''Flagcmd: Flag one solution using one command in a list'''
flist = ["mode='clip' clipminmax=[0,3] correlation='REAL_Sol1' datacolumn='CPARAM'"]
flagcmd(vis=self.vis, inpmode='list', inpfile=flist)
res = flagdata(vis=self.vis, mode='summary')
self.assertEqual(res['flagged'], 309388)
self.assertEqual(res['correlation']['Sol2']['flagged'], 0)
def test_vla_mixed_polarizations_mms2(self):
self.outputms = 'test_vla_mixed_polarizations_2.mms'
mstransform(vis=self.vis,outputvis=self.outputms,scan='16',datacolumn='DATA', createmms=True,
separationaxis='spw',spw='16~18',correlation='XX')
# Check that DDI sub-table is consistent with POLARIZATION sub-table
mytb = tbtool()
mytb.open(self.outputms + '/POLARIZATION')
npols = mytb.nrows()
mytb.close()
mytb = tbtool()
mytb.open(self.outputms + '/DATA_DESCRIPTION')
polIds = mytb.getcol('POLARIZATION_ID')
mytb.close()
for id in polIds:
self.assertTrue(id in range(npols),'PolarizationId in DATA_DESCRIPTION not consistent with POLARIZATION table')
# self.assertTrue(all(polIds < npols), 'PolarizationId in DATA_DESCRIPTION not consistent with POLARIZATION table')
# Check that flagdata can run properly with output MS
summary = flagdata(vis=self.outputms,mode='summary')
self.assertTrue(summary.has_key('correlation'), 'Flagdata failure due to missformated MS')
def test_apply1(self):
"""importevla: apply all flags and save to file"""
msname = "online.ms"
cmdfile = "online_cmd.txt"
if os.path.exists(msname):
os.system("rm -rf " + msname)
if os.path.exists(cmdfile):
os.system("rm -rf " + cmdfile)
importevla(
asdm=self.asdm,
vis=msname,
scans="2",
online=True,
applyflags=True,
shadow=True,
flagzero=True,
flagbackup=False,
savecmds=True,
)
# Check flags
res = flagdata(vis=msname, mode="summary")
self.assertEqual(res["flagged"], 2446080)
# Check output file existence
self.assertTrue(os.path.exists(cmdfile))
# Check file content
ff = open(cmdfile, "r")
cmdlist = ff.readlines()
ncmds = cmdlist.__len__()
self.assertEqual(ncmds, 216)
def test_list_reason1(self):
'''flagcmd: select by reason from two files'''
self.setUp_ngc5921()
# creat first input file
myinput = "scan='1' spw='0:10~20' reason='NONE'\n"\
"scan='2' reason='EVEN'\n"\
"scan='3' reason='ODD'"
filename1 = 'reasonfile1.txt'
create_input1(myinput, filename1)
# Create second input file
myinput = "scan='5' reason='ODD'\n"\
"scan='6' reason='EVEN'\n"\
"scan='7' reason='ODD'"
filename2 = 'reasonfile2.txt'
create_input1(myinput, filename2)
# Apply flag cmds on ODD reason
flagcmd(vis=self.vis, inpmode='list', inpfile=[filename1,filename2], reason='ODD',
flagbackup=False, action='apply')
res = flagdata(vis=self.vis, mode='summary')
self.assertEqual(res['scan']['3']['flagged'], 762048)
self.assertEqual(res['scan']['5']['flagged'], 142884)
self.assertEqual(res['scan']['6']['flagged'], 0)
self.assertEqual(res['scan']['7']['flagged'], 190512)
self.assertEqual(res['flagged'], 762048+142884+190512)
def test_apply1(self):
'''importevla: apply all flags and save to file'''
msname = 'online.ms'
cmdfile = 'online_cmd.txt'
if os.path.exists(msname):
os.system('rm -rf ' + msname)
if os.path.exists(cmdfile):
os.system('rm -rf ' + cmdfile)
importevla(asdm=self.asdm,
vis=msname,
scans='2',
online=True,
applyflags=True,
shadow=True,
flagzero=True,
flagbackup=False,
savecmds=True)
# Check flags
res = flagdata(vis=msname, mode='summary')
self.assertEqual(res['flagged'], 2446080)
# Check output file existence
self.assertTrue(os.path.exists(cmdfile))
# Check file content
ff = open(cmdfile, 'r')
cmdlist = ff.readlines()
ncmds = cmdlist.__len__()
self.assertEqual(ncmds, 216)
def test_umanualflag(self):
'''flagcmd: unapply manual agent'''
# Remove any cmd from table
flagcmd(vis=self.vis, action='clear', clearall=True)
# Flag using manual agent
myinput = "scan='1'"
filename = create_input(myinput)
flagcmd(vis=self.vis, inpmode='list', inpfile=filename, action='apply', savepars=True,
flagbackup=False)
# Flag using the quack agent
myinput = "scan='1~3' mode=quack quackinterval=1.0"
filename = create_input(myinput)
flagcmd(vis=self.vis, inpmode='list', inpfile=filename, action='apply', savepars=True,
flagbackup=False)
# Unapply the manual line
flagcmd(vis=self.vis, action='unapply', useapplied=True, tablerows=0, savepars=False)
result = flagdata(vis=self.vis,mode='summary',scan='1,2,3')
# scan 1 should be fully unflagged
self.assertEqual(result['scan']['1']['flagged'], 0)
self.assertEqual(result['scan']['2']['flagged'], 47628)
self.assertEqual(result['scan']['3']['flagged'], 47628)
self.assertEqual(result['flagged'], 47628+47628)
def test_CAS4819(self):
'''flagcmd: CAS-4819, Flag commands from three files'''
self.setUp_ngc5921()
# creat first input file
myinput = "scan='1'\n"\
"scan='2'\n"\
"scan='3'"
filename1 = 'list7a.txt'
create_input1(myinput, filename1)
# Create second input file
myinput = "scan='5'\n"\
"scan='6'\n"\
"scan='7'"
filename2 = 'list7b.txt'
create_input1(myinput, filename2)
# Create third input file
myinput = "scan='4' mode='clip' clipminmax=[0,4]"
filename3 = 'list7c.txt'
create_input1(myinput, filename3)
flagcmd(vis=self.vis, inpmode='list', inpfile=[filename1,filename2,filename3],
flagbackup=False, action='apply')
res = flagdata(vis=self.vis, mode='summary')
self.assertEqual(res['scan']['1']['flagged'], 568134)
self.assertEqual(res['scan']['2']['flagged'], 238140)
self.assertEqual(res['scan']['3']['flagged'], 762048)
self.assertEqual(res['scan']['4']['flagged'], 6696)
self.assertEqual(res['scan']['5']['flagged'], 142884)
self.assertEqual(res['scan']['6']['flagged'], 857304)
self.assertEqual(res['scan']['7']['flagged'], 190512)
self.assertEqual(res['total'],2854278)
self.assertEqual(res['flagged'],2765718)
def test_compatibility(self):
myinput = "observation='1' mode='manualflag'"
filename = create_input(myinput)
flagcmd(vis=self.vis, inpmode='list', inpfile=filename, action='apply', savepars=False,
flagbackup=False)
test_eq(flagdata(vis=self.vis, mode='summary'), 2882778, 28500)
def test1m(self):
'''flagmanager test1m: mode=list, flagbackup=True/False'''
self.unflag_ms()
flagmanager(vis=self.vis, mode='list')
aflocal.open(self.vis)
self.assertEqual(len(aflocal.getflagversionlist()), 2)
aflocal.done()
flagdata(vis=self.vis, mode='unflag', flagbackup=False)
flagmanager(vis=self.vis, mode='list')
aflocal.open(self.vis)
self.assertEqual(len(aflocal.getflagversionlist()), 2)
aflocal.done()
flagdata(vis=self.vis, mode='unflag', flagbackup=True)
flagmanager(vis=self.vis, mode='list')
aflocal.open(self.vis)
self.assertEqual(len(aflocal.getflagversionlist()), 3)
aflocal.done()
newname = 'Ha! The best version ever!'
flagmanager(vis=self.vis,
mode='rename',
oldname='flagdata_1',
versionname=newname,
comment='This is a *much* better name')
flagmanager(vis=self.vis, mode='list')
aflocal.open(self.vis)
self.assertEqual(len(aflocal.getflagversionlist()), 3)
aflocal.done()
self.assertTrue(
os.path.exists(self.vis + '.flagversions/flags.' + newname),
'Flagversion file does not exist: flags.' + newname)
# Specific for MMS
if testmms:
areg = self.vis + '/SUBMSS/*flagversions*'
import glob
print 'Check for .flagversions in the wrong place.'
self.assertEqual(
glob.glob(areg), [],
'There should not be any .flagversions in the'
' SUBMSS directory')
def test_cal_observation(self):
'''Flagcmd: flag an observation from an old cal table format'''
# Note: this cal table does not have an observation column.
# The column and sub-table should be added and the flagging
# should happen after this.
flagcmd(vis=self.vis, inpmode='list', inpfile=["observation='0'"], flagbackup=False)
res=flagdata(vis=self.vis, mode='summary')
self.assertEqual(res['flagged'],1248000)
self.assertEqual(res['total'],1248000)
def test_rflagauto(self):
"""flagcmd:: Test of rflag with defaults
"""
# (6) flagcmd AUTO. Should give same answers as test_flagdata[test_rflag1]
flagdata(vis=self.vis,mode='unflag');
flagcmd(vis=self.vis, inpmode='list', inpfile=['mode=rflag spw=9,10 extendflags=False'],
action='apply',
flagbackup=False)
res6 = flagdata(vis=self.vis, mode='summary')
print "(6) Finished flagcmd test : auto : ", res6['flagged']
#(7) flagdata AUTO (same as test_flagdata[test_rflag1])
#flagdata(vis=self.vis,mode='unflag');
#flagdata(vis=self.vis, mode='rflag', spw='9,10');
#res7 = flagdata(vis=self.vis, mode='summary')
#print "\n---------------------- Finished flagdata test : auto : ", res7['flagged']
self.assertEqual(res6['flagged'], 42728.0)
def test_xml2(self):
'''flagcmd: list xml file and save in outfile'''
# The MS only contains clip and shadow commands
# Apply the shadow command
flagcmd(vis=self.vis, action='apply', reason='SHADOW', flagbackup=False)
res = flagdata(vis=self.vis, mode='summary')
self.assertEqual(res['flagged'], 240640)
def test_apply5(self):
'''importevla: Apply only shadow flags'''
msname = 'xosro_shadow.ms'
importevla(asdm=self.asdm, vis=msname,online=False, shadow=True, flagzero=False,
applyflags=True,savecmds=False, flagbackup=False)
# This data set doesn't have shadow. Maybe change later!
res = flagdata(vis=msname, mode='summary')
self.assertEqual(res['flagged'],0,'There are shadowed antenna in this data set')
def test_apply5(self):
'''importevla: Apply only shadow flags'''
msname = 'xosro_shadow.ms'
importevla(asdm=self.asdm, vis=msname,online=False, shadow=True, flagzero=False,
applyflags=True,savecmds=False, flagbackup=False)
# This data set doesn't have shadow. Maybe change later!
res = flagdata(vis=msname, mode='summary')
self.assertEqual(res['flagged'],0,'There are shadowed antenna in this data set')
def test_rename(self):
'''flagmanager: do not overwrite an existing versionname'''
# Create a flagbackup
flagdata(vis=self.vis, mode='manual', antenna="2", flagbackup=True)
fname = 'flagdata_1'
self.assertTrue(os.path.exists(self.vis+'.flagversions/flags.'+fname),
'Flagversions file does not exist: flags.'+fname)
# Rename it
newname = 'Rename_Me'
flagmanager(vis=self.vis, mode='rename', oldname=fname, versionname=newname,
comment='CAS-3080')
self.assertTrue(os.path.exists(self.vis+'.flagversions/flags.'+newname),
'Flagversions file does not exist: flags.'+newname)
self.assertFalse(os.path.exists(self.vis+'.flagversions/flags.'+fname),
'Flagversions file shuold not exist: flags.'+fname)
def test_list1(self):
'''flagcmd: list and savepars=True/False'''
# Remove any cmd from table
flagcmd(vis=self.vis, action='clear', clearall=True)
########## TEST 1
# create text file called flagcmd.txt
myinput = "scan='4' mode='clip' correlation='ABS_RR' clipminmax=[0, 4]\n"
filename = create_input(myinput)
filename1 = 'filename1.txt'
os.system('cp '+filename+' '+filename1)
# save command to MS
flagcmd(vis=self.vis, action='list', inpmode='list', inpfile=[myinput], savepars=True)
# list/save to a file
os.system('rm -rf myflags.txt')
flagcmd(vis=self.vis, action='list', outfile='myflags.txt', savepars=True)
# compare saved file with original input file
self.assertTrue(filecmp.cmp(filename1, 'myflags.txt', 1), 'Files should be equal')
########## TEST 2
# create another input
myinput = "scan='1~3' mode='manual'\n"
filename = create_input(myinput)
# apply and don't save to MS
flagcmd(vis=self.vis, inpmode='list', inpfile=filename, action='apply', savepars=False,
flagbackup=False)
# list and check that parameters were not saved to MS
os.system('rm -rf myflags.txt')
flagcmd(vis=self.vis, action='list', outfile='myflags.txt', savepars=True)
self.assertFalse(filecmp.cmp(filename, 'myflags.txt', 1), 'Files should not be equal')
########### TEST 3
# apply cmd from TEST 1 and update APPLIED column
flagcmd(vis=self.vis, action='apply', savepars=False, flagbackup=False)
# scans=1~3 should be fully flagged
res = flagdata(vis=self.vis, mode='summary')
self.assertEqual(res['scan']['1']['flagged'], 568134)
self.assertEqual(res['scan']['1']['total'], 568134)
self.assertEqual(res['scan']['2']['flagged'], 238140)
self.assertEqual(res['scan']['2']['total'], 238140)
self.assertEqual(res['scan']['3']['flagged'], 762048)
self.assertEqual(res['scan']['3']['total'], 762048)
# scan 4 should be partially flagged by clip mode
self.assertEqual(res['scan']['4']['flagged'], 3348, 'Only RR should be flagged')
self.assertEqual(res['scan']['4']['total'], 95256)
# Only cmd from TEST 1 should be in MS
os.system('rm -rf myflags.txt')
flagcmd(vis=self.vis, action='list', outfile='myflags.txt', useapplied=True, savepars=True)
self.assertTrue(filecmp.cmp(filename1, 'myflags.txt', 1), 'Files should be equal')
def test_caltable_flagbackup(self):
'''Flagmanager:: cal table mode=list, flagbackup=True/False'''
# Need a fresh start
os.system('rm -rf cal.fewscans.bpass*')
self.setUp_bpass_case()
self.unflag_ms()
flagmanager(vis=self.vis, mode='list')
aflocal.open(self.vis)
self.assertEqual(len(aflocal.getflagversionlist()), 2)
aflocal.done()
flagdata(vis=self.vis, mode='unflag', flagbackup=False)
flagmanager(vis=self.vis, mode='list')
aflocal.open(self.vis)
self.assertEqual(len(aflocal.getflagversionlist()), 2)
aflocal.done()
flagdata(vis=self.vis, mode='unflag', flagbackup=True)
flagmanager(vis=self.vis, mode='list')
aflocal.open(self.vis)
self.assertEqual(len(aflocal.getflagversionlist()), 3)
aflocal.done()
newname = 'A much better caltable version !'
flagmanager(vis=self.vis,
mode='rename',
oldname='flagdata_1',
versionname=newname,
comment='This is a *much* better name')
flagsdict = flagmanager(vis=self.vis, mode='list')
aflocal.open(self.vis)
self.assertEqual(len(aflocal.getflagversionlist()), 3)
aflocal.done()
self.assertTrue(
os.path.exists(self.vis + '.flagversions/flags.' + newname),
'Flagversion file does not exist: flags.' + newname)
self.assertEqual(flagsdict[0]['name'], newname,
'Dictionary should show new name of flag versions')
print flagsdict
def test_apply4(self):
'''importevla: Save online flags to FLAG_CMD and file; do not apply'''
# Use default msname and outfile
msname = 'xosro4.ms'
cmdfile = msname.replace('.ms','_cmd.txt')
if os.path.exists(msname):
os.system('rm -rf '+msname)
if os.path.exists(cmdfile):
os.system('rm -rf '+cmdfile)
importevla(asdm=self.asdm, vis=msname, scans='2',online=True, shadow=False, flagzero=False,
applyflags=False,savecmds=True, flagbackup=False)
# No flags were applied
res = flagdata(vis=msname, mode='summary')
self.assertEqual(res['flagged'],0)
# Apply only row 213 using flagcmd
# The command in row 213 is the following:
# antenna='ea06' timerange='2012/02/22/22:30:55.200~2012/02/22/22:35:08.199'
# spw='EVLA_X#A0C0#0' correlation='LL,LR,RL
flagcmd(vis=msname, action='apply', tablerows=213)
# Check flags. RR should no be flagged
res = flagdata(vis=msname, mode='summary')
self.assertEqual(res['correlation']['RR']['flagged'],0,'RR should not be flagged')
self.assertEqual(res['correlation']['LL']['flagged'],29440)
self.assertEqual(res['correlation']['LR']['flagged'],29440)
self.assertEqual(res['correlation']['RL']['flagged'],29440)
self.assertEqual(res['antenna']['ea06']['flagged'],88320)
self.assertEqual(res['antenna']['ea07']['flagged'],3840,'Only a few baselines should be flagged')
self.assertEqual(res['antenna']['ea08']['flagged'],3840,'Only a few baselines should be flagged')
# Check output file existence
self.assertTrue(os.path.exists(cmdfile))
# Check file content
ff = open(cmdfile,'r')
cmdlist = ff.readlines()
ncmds = cmdlist.__len__()
ff.close()
self.assertEqual(ncmds, 214, 'Only Online cmds should have been saved to file')
def test_manual_field_selection_for_bpass(self):
"""Flagcmd:: Manually flag a bpass-based CalTable using field selection"""
flagcmd(vis=self.vis, inpmode='list', inpfile=["field='3C286_A'"], flagbackup=False)
summary=flagdata(vis=self.vis, mode='summary')
self.assertEqual(summary['field']['3C286_A']['flagged'], 499200.0)
self.assertEqual(summary['field']['3C286_B']['flagged'], 0)
self.assertEqual(summary['field']['3C286_C']['flagged'], 0)
self.assertEqual(summary['field']['3C286_D']['flagged'], 0)
def test_flagbackup(self):
'''Flagcmd: backup cal table flags'''
flagmanager(vis=self.vis, mode='list')
aflocal.open(self.vis)
self.assertEqual(len(aflocal.getflagversionlist()), 2)
aflocal.done()
flagcmd(vis=self.vis, inpmode='list', inpfile=["spw='3'"])
flagmanager(vis=self.vis, mode='list')
aflocal.open(self.vis)
self.assertEqual(len(aflocal.getflagversionlist()), 3)
aflocal.done()
flagcmd(vis=self.vis, inpmode='list', inpfile=["spw='4'"], flagbackup=False)
flagmanager(vis=self.vis, mode='list')
aflocal.open(self.vis)
self.assertEqual(len(aflocal.getflagversionlist()), 3)
aflocal.done()
newname = 'BackupBeforeSpwFlags'
flagmanager(vis=self.vis, mode='rename', oldname='flagcmd_1', versionname=newname,
comment='Backup of flags before applying flags on spw')
flagmanager(vis=self.vis, mode='list')
aflocal.open(self.vis)
self.assertEqual(len(aflocal.getflagversionlist()), 3)
aflocal.done()
# Apply spw=5 flags
flagcmd(vis=self.vis, inpmode='list', inpfile=["spw='5'"], flagbackup=True)
res = flagdata(vis=self.vis, mode='summary')
self.assertEqual(res['spw']['3']['flagged'], 83200)
self.assertEqual(res['spw']['4']['flagged'], 83200)
self.assertEqual(res['spw']['5']['flagged'], 83200)
self.assertEqual(res['flagged'], 83200*3)
# Restore backup
flagmanager(vis=self.vis, mode='restore', versionname='BackupBeforeSpwFlags', merge='replace')
res = flagdata(vis=self.vis, mode='summary')
self.assertEqual(res['spw']['5']['flagged'], 0)
self.assertEqual(res['flagged'], 0)
def test_action_list1(self):
'''flagcmd: action=list with inpmode from a list'''
flagcmd(vis=self.vis, action='clear', clearall=True)
cmd = ["spw='5~7'","spw='1'"]
flagcmd(vis=self.vis, action='list', inpmode='list', inpfile=cmd, savepars=True)
# Apply the flags
flagcmd(vis=self.vis)
res=flagdata(vis=self.vis, mode='summary')
self.assertEqual(res['flagged'],1099776)
