def test_flagversions(self):
'''Partition: check that the .flagversions is created'''
# Run partition and create the .flagversions
partition(vis=self.msfile,
outputvis=self.mmsfile,
createmms=True,
disableparallel=True,
datacolumn='data')
self.assertTrue(os.path.exists(self.mmsfile + '.flagversions'))
# Check that the number of backups in MMS is correct
aflocal = aftool()
aflocal.open(self.mmsfile)
nv = aflocal.getflagversionlist()
aflocal.done()
self.assertEqual(len(nv), 3)
# Run flagdata on MMS to check if it works well.
flagdata(vis=self.mmsfile, mode='unflag', flagbackup=True)
# Check that the number of backups in MMS is correct
aflocal = aftool()
aflocal.open(self.mmsfile)
nvref = aflocal.getflagversionlist()
aflocal.done()
self.assertEqual(len(nvref), 4)
def test_manual_field_msSelection_layer_for_tsys_CalTable(self):
"""AgentFlagger:: Manually flag a Tsys-based CalTable using MSSelection for field """
aflocal = aftool()
aflocal.open(self.vis)
aflocal.selectdata(field='0')
agentUnflag = {'apply': True, 'mode': 'unflag'}
agentManual = {'apply': True, 'mode': 'manual'}
aflocal.parseagentparameters(agentUnflag)
aflocal.parseagentparameters(agentManual)
aflocal.init()
summary = aflocal.run(writeflags=True)
aflocal.done()
aflocal.open(self.vis)
aflocal.selectdata()
agentSummary = {'apply': True, 'mode': 'summary'}
aflocal.parseagentparameters(agentSummary)
aflocal.init()
summary = aflocal.run(writeflags=True)
aflocal.done()
self.assertEqual(summary['report0']['field']['3c279']['flagged'],
9216.0)
self.assertEqual(summary['report0']['field']['Titan']['flagged'], 0)
self.assertEqual(summary['report0']['field']['TW Hya']['flagged'], 0)
self.assertEqual(
summary['report0']['field']['J1037-295=QSO']['flagged'], 0)
def test_clip_nan_and_inf_fparm_all(self):
"""AgentFlagger:: Test cliping only NaNs/Infs in all calibration solution products of FPARAM column"""
aflocal = aftool()
datacolumn = 'FPARAM'
correlation = ''
aflocal.open(self.vis)
aflocal.selectdata()
agentUnflag = {'apply': True, 'mode': 'unflag'}
agentClip = {
'apply': True,
'mode': 'clip',
'datacolumn': datacolumn,
'correlation': correlation
}
agentSummary = {'apply': True, 'mode': 'summary'}
aflocal.parseagentparameters(agentUnflag)
aflocal.parseagentparameters(agentClip)
aflocal.parseagentparameters(agentSummary)
aflocal.init()
summary = aflocal.run(writeflags=True)
aflocal.done()
self.assertEqual(summary['report0']['total'], 129024.0)
self.assertEqual(summary['report0']['flagged'], 0.0)
self.assertEqual(summary['report0']['correlation']['Sol1']['flagged'],
0.0)
self.assertEqual(summary['report0']['correlation']['Sol1']['total'],
64512.0)
self.assertEqual(summary['report0']['correlation']['Sol2']['flagged'],
0.0)
self.assertEqual(summary['report0']['correlation']['Sol2']['total'],
64512.0)
def test_clip_minmax_fparm_all(self):
"""AgentFlagger:: Test cliping all calibration solution products of FPARAM column using a minmax range """
aflocal = aftool()
datacolumn = 'FPARAM'
correlation = ''
aflocal.open(self.vis)
aflocal.selectdata()
agentUnflag = {'apply': True, 'mode': 'unflag'}
agentClip = {
'apply': True,
'mode': 'clip',
'clipzeros': True,
'clipminmax': [0., 600.],
'datacolumn': datacolumn,
'correlation': correlation
}
agentSummary = {'apply': True, 'mode': 'summary'}
aflocal.parseagentparameters(agentUnflag)
aflocal.parseagentparameters(agentClip)
aflocal.parseagentparameters(agentSummary)
aflocal.init()
summary = aflocal.run(writeflags=True)
aflocal.done()
self.assertEqual(summary['report0']['total'], 129024.0)
self.assertEqual(summary['report0']['flagged'], 1192.0)
self.assertEqual(summary['report0']['correlation']['Sol1']['flagged'],
750.0)
self.assertEqual(summary['report0']['correlation']['Sol1']['total'],
64512.0)
self.assertEqual(summary['report0']['correlation']['Sol2']['flagged'],
442.0)
self.assertEqual(summary['report0']['correlation']['Sol2']['total'],
64512.0)
def test_manual_field_msSelection_layer_for_bpass_CalTable(self):
"""AgentFlagger:: Manually flag a bpass-based CalTable using MSSelection for field """
aflocal = aftool()
aflocal.open(self.vis)
aflocal.selectdata({'field': '3C286_A'})
agentUnflag = {'apply': True, 'mode': 'unflag'}
agentManual = {'apply': True, 'mode': 'manual'}
aflocal.parseagentparameters(agentUnflag)
aflocal.parseagentparameters(agentManual)
aflocal.init()
summary = aflocal.run(writeflags=True)
aflocal.done()
aflocal.open(self.vis)
aflocal.selectdata()
agentSummary = {'apply': True, 'mode': 'summary'}
aflocal.parseagentparameters(agentSummary)
aflocal.init()
summary = aflocal.run(writeflags=True)
aflocal.done()
self.assertEqual(summary['report0']['field']['3C286_A']['flagged'],
499200.0)
self.assertEqual(summary['report0']['field']['3C286_B']['flagged'], 0)
self.assertEqual(summary['report0']['field']['3C286_C']['flagged'], 0)
self.assertEqual(summary['report0']['field']['3C286_D']['flagged'], 0)
def test_clip_minmax_fparm_error_case_absall(self):
"""AgentFlagger:: Error case test when a complex operator is used with CalTables """
aflocal = aftool()
datacolumn = 'FPARAM'
correlation = 'ABS_ALL'
aflocal.open(self.vis)
aflocal.selectdata()
agentUnflag = {'apply': True, 'mode': 'unflag'}
agentClip = {
'apply': True,
'mode': 'clip',
'clipzeros': True,
'clipminmax': [0., 600.],
'datacolumn': datacolumn,
'correlation': correlation
}
agentSummary = {'apply': True, 'mode': 'summary'}
aflocal.parseagentparameters(agentUnflag)
aflocal.parseagentparameters(agentClip)
aflocal.parseagentparameters(agentSummary)
aflocal.init()
summary = aflocal.run(writeflags=True)
aflocal.done()
self.assertEqual(summary['report0']['total'], 129024.0)
self.assertEqual(summary['report0']['flagged'], 1192.0)
self.assertEqual(summary['report0']['correlation']['Sol1']['flagged'],
750.0)
self.assertEqual(summary['report0']['correlation']['Sol1']['total'],
64512.0)
self.assertEqual(summary['report0']['correlation']['Sol2']['flagged'],
442.0)
self.assertEqual(summary['report0']['correlation']['Sol2']['total'],
64512.0)
def test_manual_antenna_msSelection_layer_for_bpass_CalTable(self):
"""AgentFlagger:: Manually flag a bpass-based CalTable using MSSelection for antenna"""
aflocal = aftool()
aflocal.open(self.vis)
aflocal.selectdata({'antenna': 'ea09'})
agentUnflag = {'apply': True, 'mode': 'unflag'}
agentManual = {'apply': True, 'mode': 'manual'}
aflocal.parseagentparameters(agentUnflag)
aflocal.parseagentparameters(agentManual)
aflocal.init()
summary = aflocal.run(writeflags=True)
aflocal.done()
aflocal.open(self.vis)
aflocal.selectdata()
agentSummary = {'apply': True, 'mode': 'summary'}
aflocal.parseagentparameters(agentSummary)
aflocal.init()
summary = aflocal.run(writeflags=True)
aflocal.done()
self.assertEqual(summary['report0']['antenna']['ea09']['flagged'],
48000.0)
self.assertEqual(summary['report0']['antenna']['ea10']['flagged'], 0.0)
def test_manual_field_msSelection_layer_for_tsys_CalTable(self):
"""AgentFlagger:: Manually flag a Tsys-based CalTable using MSSelection for field """
aflocal = aftool()
aflocal.open(self.vis)
aflocal.selectdata(field='0')
agentUnflag={'apply':True,'mode':'unflag'}
agentManual={'apply':True,'mode':'manual'}
aflocal.parseagentparameters(agentUnflag)
aflocal.parseagentparameters(agentManual)
aflocal.init()
summary = aflocal.run(writeflags=True)
aflocal.done()
aflocal.open(self.vis)
aflocal.selectdata()
agentSummary={'apply':True,'mode':'summary'}
aflocal.parseagentparameters(agentSummary)
aflocal.init()
summary = aflocal.run(writeflags=True)
aflocal.done()
self.assertEqual(summary['report0']['field']['3c279']['flagged'], 9216.0)
self.assertEqual(summary['report0']['field']['Titan']['flagged'], 0)
self.assertEqual(summary['report0']['field']['TW Hya']['flagged'], 0)
self.assertEqual(summary['report0']['field']['J1037-295=QSO']['flagged'], 0)
def test_clip_minmax_snr_all_for_bpass_CalTable(self):
"""AgentFlagger:: Test cliping all calibration solution products of SNR column using a minmax range for bpass CalTable"""
aflocal = aftool()
datacolumn = 'SNR'
correlation = ''
aflocal.open(self.vis)
aflocal.selectdata()
agentUnflag = {'apply': True, 'mode': 'unflag'}
agentClip = {
'apply': True,
'mode': 'clip',
'clipzeros': True,
'clipminmax': [0., 550.],
'datacolumn': datacolumn,
'correlation': correlation
}
agentSummary = {'apply': True, 'mode': 'summary'}
aflocal.parseagentparameters(agentUnflag)
aflocal.parseagentparameters(agentClip)
aflocal.parseagentparameters(agentSummary)
aflocal.init()
summary = aflocal.run(writeflags=True)
aflocal.done()
self.assertEqual(summary['report0']['total'], 1248000.0)
self.assertEqual(summary['report0']['flagged'], 74371.0)
self.assertEqual(summary['report0']['correlation']['Sol1']['flagged'],
36327.0)
self.assertEqual(summary['report0']['correlation']['Sol1']['total'],
624000.0)
self.assertEqual(summary['report0']['correlation']['Sol2']['flagged'],
38044.0)
self.assertEqual(summary['report0']['correlation']['Sol2']['total'],
624000.0)
def test_clip_minmax_fparm_sol1_2(self):
"""AgentFlagger:: Test cliping first calibration solution product of FPARAM column using a minmax range """
# Run the previous test but using the specific parsing functions
aflocal = aftool()
datacolumn = 'FPARAM'
correlation = 'Sol1'
aflocal.open(self.vis)
aflocal.selectdata()
aflocal.parsemanualparameters(apply=False)
aflocal.parseclipparameters(clipzeros=True,
clipminmax=[0., 600.],
datacolumn=datacolumn,
correlation=correlation)
aflocal.parsesummaryparameters()
aflocal.init()
summary = aflocal.run(writeflags=True)
aflocal.done()
self.assertEqual(summary['report0']['total'], 129024)
self.assertEqual(summary['report0']['flagged'], 750)
self.assertEqual(summary['report0']['correlation']['Sol1']['flagged'],
750)
self.assertEqual(summary['report0']['correlation']['Sol1']['total'],
64512)
self.assertEqual(summary['report0']['correlation']['Sol2']['flagged'],
0)
self.assertEqual(summary['report0']['correlation']['Sol2']['total'],
64512)
def test_manual_field_msSelection_layer_for_bpass_CalTable(self):
"""AgentFlagger:: Manually flag a bpass-based CalTable using MSSelection for field """
aflocal = aftool()
aflocal.open(self.vis)
aflocal.selectdata({'field':'3C286_A'})
agentUnflag={'apply':True,'mode':'unflag'}
agentManual={'apply':True,'mode':'manual'}
aflocal.parseagentparameters(agentUnflag)
aflocal.parseagentparameters(agentManual)
aflocal.init()
summary = aflocal.run(writeflags=True)
aflocal.done()
aflocal.open(self.vis)
aflocal.selectdata()
agentSummary={'apply':True,'mode':'summary'}
aflocal.parseagentparameters(agentSummary)
aflocal.init()
summary = aflocal.run(writeflags=True)
aflocal.done()
self.assertEqual(summary['report0']['field']['3C286_A']['flagged'], 499200.0)
self.assertEqual(summary['report0']['field']['3C286_B']['flagged'], 0)
self.assertEqual(summary['report0']['field']['3C286_C']['flagged'], 0)
self.assertEqual(summary['report0']['field']['3C286_D']['flagged'], 0)
def test_summarylist1(self):
'''agentflagger: multiple summaries'''
aflocal = aftool()
aflocal.open(self.vis)
aflocal.selectdata()
aflocal.parsesummaryparameters(name='summary_1')
aflocal.parsemanualparameters(intent='FOCUS') # non-existing intent
aflocal.parsesummaryparameters(name='summary_2')
aflocal.parsemanualparameters(spw='0')
aflocal.parsesummaryparameters(name='summary_3')
aflocal.init()
summary = aflocal.run()
aflocal.done()
# It creates 2 reports per summary agent. One is type='summary',
# the other is type='plotpoints'
# jagonzal: Not anymore, now it creates only 1 report per summary
# agent unless display is 'report' or 'both'
self.assertEqual(summary['report0']['name'], 'summary_1')
self.assertEqual(summary['report1']['name'], 'summary_2')
self.assertEqual(summary['report2']['name'], 'summary_3')
self.assertEqual(summary['report0']['flagged'], 0)
self.assertEqual(summary['report1']['flagged'], 0)
self.assertEqual(summary['report2']['spw']['0']['flagged'],
summary['report2']['spw']['0']['total'])
def test_display_cal_tables(self):
'''AgentFlagger: Select spws, display and flag cal tables'''
self.setUp_bpass_case()
aflocal = aftool()
aflocal.open(self.vis)
aflocal.selectdata(spw='1')
aflocal.parsemanualparameters(spw='1:0~10;60~63')
aflocal.parsesummaryparameters(spwchan=True)
# agentManual={'apply':True,'mode':'manual'}
# agentSummary={'apply':True,'mode':'summary','spwchan':True}
agentDisplay={'mode':'display','datadisplay':True,'pause':False,'datacolumn':'CPARAM'}
# aflocal.parseagentparameters(agentManual)
# aflocal.parseagentparameters(agentSummary)
aflocal.parseagentparameters(agentDisplay)
aflocal.init()
summary = aflocal.run(writeflags=True)
aflocal.done()
self.assertEqual(summary['report0']['spw:channel']['1:0']['flagged'], 1300)
self.assertEqual(summary['report0']['spw:channel']['1:5']['flagged'], 1300)
self.assertEqual(summary['report0']['spw:channel']['1:10']['flagged'], 1300)
self.assertEqual(summary['report0']['spw:channel']['1:11']['flagged'], 0)
self.assertEqual(summary['report0']['spw:channel']['1:60']['flagged'], 1300)
self.assertEqual(summary['report0']['spw:channel']['1:59']['flagged'], 0)
self.assertEqual(summary['report0']['flagged'], 15*1300)
def test_tfcrop_cparam_sol1_extension(self):
"""AgentFlagger:: Test tfcrop first calibration solution product of CPARAM column,
and then extend to the other solution for bpass CalTable"""
aflocal = aftool()
datacolumn = "CPARAM"
correlation = 'Sol1'
aflocal.open(self.vis)
aflocal.selectdata()
# Pre-clip data to avoid problems with near-zero values
agentClip={'apply':True,'mode':'clip','clipzeros':True,'datacolumn':datacolumn,'correlation':correlation}
aflocal.parseagentparameters(agentClip)
aflocal.parsetfcropparameters(datacolumn=datacolumn, correlation=correlation,
extendflags=False)
aflocal.parsesummaryparameters()
# Extend to the other solution
aflocal.parseextendparameters(extendpols=True,growfreq=0.0,growtime=0.0)
aflocal.parsesummaryparameters()
aflocal.init()
summary = aflocal.run(writeflags=True)
aflocal.done()
# flags from first summary, only tfcrop
assert abs(summary['report0']['flagged'] - 30427) <= 5
assert abs(summary['report0']['correlation']['Sol1']['flagged'] - 30427) <= 5
# flags from second summary, tfcrop+extend
assert abs(summary['report1']['flagged'] - 2*30427) <= 10
assert abs(summary['report1']['correlation']['Sol2']['flagged'] - 30427) <= 5
def test_display_cal_tables(self):
'''AgentFlagger: Select spws, display and flag cal tables'''
self.setUp_bpass_case()
aflocal = aftool()
aflocal.open(self.vis)
aflocal.selectdata(spw='1')
aflocal.parsemanualparameters(spw='1:0~10;60~63')
aflocal.parsesummaryparameters(spwchan=True)
# agentManual={'apply':True,'mode':'manual'}
# agentSummary={'apply':True,'mode':'summary','spwchan':True}
agentDisplay={'mode':'display','datadisplay':True,'pause':False,'datacolumn':'CPARAM'}
# aflocal.parseagentparameters(agentManual)
# aflocal.parseagentparameters(agentSummary)
aflocal.parseagentparameters(agentDisplay)
aflocal.init()
summary = aflocal.run(writeflags=True)
aflocal.done()
self.assertEqual(summary['report0']['spw:channel']['1:0']['flagged'], 1300)
self.assertEqual(summary['report0']['spw:channel']['1:5']['flagged'], 1300)
self.assertEqual(summary['report0']['spw:channel']['1:10']['flagged'], 1300)
self.assertEqual(summary['report0']['spw:channel']['1:11']['flagged'], 0)
self.assertEqual(summary['report0']['spw:channel']['1:60']['flagged'], 1300)
self.assertEqual(summary['report0']['spw:channel']['1:59']['flagged'], 0)
self.assertEqual(summary['report0']['flagged'], 15*1300)
def test_antint_simple_for_bpass_CalTable(self):
'''Test the antenna integrations flagging mode on a bpass-based CalTable
using MSSelection for antenna'''
aflocal = aftool()
aflocal.open(self.vis)
aflocal.selectdata()
agentUnflag = {'apply':True, 'mode':'unflag'}
agentAntInt = {'apply': True, 'mode': 'antint', 'datacolumn': 'CPARAM',
'antint_ref_antenna': 'ea01', 'spw': '0', 'field': '0',
'minchanfrac': 0.7, 'verbose': True}
agentSummary={'apply':True,'mode':'summary'}
aflocal.parseagentparameters(agentUnflag)
aflocal.parseagentparameters(agentAntInt)
aflocal.parsesummaryparameters()
aflocal.parseagentparameters(agentSummary)
aflocal.init()
summary = aflocal.run(writeflags=True)
aflocal.done()
self.assertEqual(summary['report0']['flagged'], 0)
self.assertEqual(summary['report0']['total'], 1248000)
self.assertEqual(summary['report0']['correlation']['Sol1']['flagged'], 0)
self.assertEqual(summary['report0']['correlation']['Sol1']['total'], 624000)
self.assertEqual(summary['report0']['correlation']['Sol2']['flagged'], 0)
self.assertEqual(summary['report0']['correlation']['Sol2']['total'], 624000)
def test_mixed_agents_tsys(self):
'''AgentFlagger: supported and unsupported agents in a list'''
aflocal = aftool()
aflocal.open(self.vis)
aflocal.selectdata()
aflocal.parsemanualparameters(spw='1')
aflocal.parseagentparameters({
'mode': 'shadow',
'spw': '3'
}) #unsupported mode
aflocal.parseelevationparameters() # unsupported mode
aflocal.parsemanualparameters(spw='5')
aflocal.parsesummaryparameters(spw='1,3,5', fieldcnt=True)
aflocal.init()
res = aflocal.run(writeflags=True)
aflocal.done()
# Check the summary dictionary with field breakdown
fields = res['report0'].keys()
fields.remove('name')
fields.remove('type')
fflags1 = 0
fflags3 = 0
fflags5 = 0
totalflags = 0
for ff in fields:
fflags1 += res['report0'][ff]['spw']['1']['flagged']
fflags3 += res['report0'][ff]['spw']['3']['flagged']
fflags5 += res['report0'][ff]['spw']['5']['flagged']
totalflags += res['report0'][ff]['flagged']
self.assertEqual(fflags1, 32256)
self.assertEqual(fflags3, 0)
self.assertEqual(fflags5, 32256)
self.assertEqual(totalflags, 32256 * 2)
def test_tfcrop_cparam_sol1_extension(self):
"""AgentFlagger:: Test tfcrop first calibration solution product of CPARAM column,
and then extend to the other solution for bpass CalTable"""
aflocal = aftool()
datacolumn = "CPARAM"
correlation = 'Sol1'
aflocal.open(self.vis)
aflocal.selectdata()
# Pre-clip data to avoid problems with near-zero values
agentClip={'apply':True,'mode':'clip','clipzeros':True,'datacolumn':datacolumn,'correlation':correlation}
aflocal.parseagentparameters(agentClip)
aflocal.parsetfcropparameters(datacolumn=datacolumn, correlation=correlation,
extendflags=False)
aflocal.parsesummaryparameters()
# Extend to the other solution
aflocal.parseextendparameters(extendpols=True,growfreq=0.0,growtime=0.0)
aflocal.parsesummaryparameters()
aflocal.init()
summary = aflocal.run(writeflags=True)
aflocal.done()
# flags from first summary, only tfcrop
assert abs(summary['report0']['flagged'] - 30427) <= 5
assert abs(summary['report0']['correlation']['Sol1']['flagged'] - 30427) <= 5
# flags from second summary, tfcrop+extend
assert abs(summary['report2']['flagged'] - 2*30427) <= 10
assert abs(summary['report2']['correlation']['Sol2']['flagged'] - 30427) <= 5
def test_mixed_agents_tsys(self):
'''AgentFlagger: supported and unsupported agents in a list'''
aflocal = aftool()
aflocal.open(self.vis)
aflocal.selectdata()
aflocal.parsemanualparameters(spw='1')
aflocal.parseagentparameters({'mode':'shadow','spw':'3'}) #unsupported mode
aflocal.parseelevationparameters() # unsupported mode
aflocal.parsemanualparameters(spw='5')
aflocal.parsesummaryparameters(spw='1,3,5', fieldcnt=True)
aflocal.init()
res = aflocal.run(writeflags=True)
aflocal.done()
# Check the summary dictionary with field breakdown
fields = res['report0'].keys()
fields.remove('name')
fields.remove('type')
fflags1 = 0
fflags3 = 0
fflags5 = 0
totalflags = 0
for ff in fields:
fflags1 += res['report0'][ff]['spw']['1']['flagged']
fflags3 += res['report0'][ff]['spw']['3']['flagged']
fflags5 += res['report0'][ff]['spw']['5']['flagged']
totalflags += res['report0'][ff]['flagged']
self.assertEqual(fflags1, 32256)
self.assertEqual(fflags3, 0)
self.assertEqual(fflags5, 32256)
self.assertEqual(totalflags, 32256*2)
def test_unsupported_elevation_tsys(self):
'''AgentFlagger: Unsupported elevation mode'''
aflocal = aftool()
aflocal.open(self.vis)
aflocal.selectdata()
aflocal.parseelevationparameters()
aflocal.init()
res = aflocal.run(writeflags=True)
aflocal.done()
self.assertEqual(res, {})
def unflag_table(self):
aflocal = aftool()
aflocal.open(self.vis)
aflocal.selectdata()
agentUnflag = {'apply': True, 'mode': 'unflag'}
aflocal.parseagentparameters(agentUnflag)
aflocal.init()
aflocal.run(writeflags=True)
aflocal.done()
def test_unsupported_elevation_tsys(self):
'''AgentFlagger: Unsupported elevation mode'''
aflocal = aftool()
aflocal.open(self.vis)
aflocal.selectdata()
aflocal.parseelevationparameters()
aflocal.init()
res = aflocal.run(writeflags=True)
aflocal.done()
self.assertEqual(res, {})
def unflag_table(self):
aflocal = aftool()
aflocal.open(self.vis)
aflocal.selectdata()
agentUnflag={'apply':True,'mode':'unflag'}
aflocal.parseagentparameters(agentUnflag)
aflocal.init()
aflocal.run(writeflags=True)
aflocal.done()
def test_default_cparam_bpass(self):
'''Flagdata: flag CPARAM data column'''
aflocal = aftool()
aflocal.open(self.vis)
aflocal.selectdata()
aflocal.parseclipparameters(clipzeros=True,datacolumn='CPARAM')
aflocal.parsesummaryparameters()
aflocal.init()
summary = aflocal.run(writeflags=True)
aflocal.done()
self.assertEqual(summary['report0']['flagged'], 11078.0, 'Should use CPARAM as the default column')
def test_default_cparam_bpass(self):
'''Flagdata: flag CPARAM data column'''
aflocal = aftool()
aflocal.open(self.vis)
aflocal.selectdata()
aflocal.parseclipparameters(clipzeros=True,datacolumn='CPARAM')
aflocal.parsesummaryparameters()
aflocal.init()
summary = aflocal.run(writeflags=True)
aflocal.done()
self.assertEqual(summary['report0']['flagged'], 11078.0, 'Should use CPARAM as the default column')
def test_display_float_data(self):
'''AgentFlagger: Select spw, display and flag single-dish MS'''
self.setUp_float_data()
aflocal = aftool()
aflocal.open(self.vis)
aflocal.selectdata(spw='1~4')
aflocal.parsemanualparameters()
aflocal.parsesummaryparameters(name='Single-dish')
agentDisplay={'mode':'display','datadisplay':True,'pause':False,'datacolumn':'FLOAT_DATA'}
aflocal.parseagentparameters(agentDisplay)
aflocal.init()
summary = aflocal.run(writeflags=True)
aflocal.done()
self.assertEqual(summary['report0']['flagged'], 2052)
def test_flagversions(self):
"""Partition: check that the .flagversions is created"""
# Run partition and create the .flagversions
partition(vis=self.msfile, outputvis=self.mmsfile, createmms=True, disableparallel=True, datacolumn="data")
self.assertTrue(os.path.exists(self.mmsfile + ".flagversions"))
# Check that the number of backups in MMS is correct
aflocal = aftool()
aflocal.open(self.mmsfile)
nv = aflocal.getflagversionlist()
aflocal.done()
self.assertEqual(len(nv), 3)
# Run flagdata on MMS to check if it works well.
flagdata(vis=self.mmsfile, mode="unflag", flagbackup=True)
# Check that the number of backups in MMS is correct
aflocal = aftool()
aflocal.open(self.mmsfile)
nvref = aflocal.getflagversionlist()
aflocal.done()
self.assertEqual(len(nvref), 4)
def test_display_float_data(self):
'''AgentFlagger: Select spw, display and flag single-dish MS'''
self.setUp_float_data()
aflocal = aftool()
aflocal.open(self.vis)
aflocal.selectdata(spw='1~4')
aflocal.parsemanualparameters()
aflocal.parsesummaryparameters(name='Single-dish')
agentDisplay={'mode':'display','datadisplay':True,'pause':False,'datacolumn':'FLOAT_DATA'}
aflocal.parseagentparameters(agentDisplay)
aflocal.init()
summary = aflocal.run(writeflags=True)
aflocal.done()
self.assertEqual(summary['report0']['flagged'], 2052)
def test_clip_minmax_cparam_for_bpass(self):
"""AgentFlagger:: Clip all calibration solutions of CPARAM column using a minmax range"""
aflocal = aftool()
datacolumn = 'CPARAM'
aflocal.open(self.vis)
aflocal.selectdata()
aflocal.parseclipparameters(clipminmax=[0.,0.3],datacolumn=datacolumn,clipzeros=True)
aflocal.parsesummaryparameters()
aflocal.init()
summary = aflocal.run(writeflags=True)
aflocal.done()
self.assertEqual(summary['report0']['flagged'], 11175.0)
self.assertEqual(summary['report0']['total'], 1248000)
self.assertEqual(summary['report0']['correlation']['Sol1']['flagged'], 11136.0)
self.assertEqual(summary['report0']['correlation']['Sol2']['flagged'], 39)
def test_clip_minmax_cparam_for_bpass(self):
"""AgentFlagger:: Clip all calibration solutions of CPARAM column using a minmax range"""
aflocal = aftool()
datacolumn = 'CPARAM'
aflocal.open(self.vis)
aflocal.selectdata()
aflocal.parseclipparameters(clipminmax=[0.,0.3],datacolumn=datacolumn,clipzeros=True)
aflocal.parsesummaryparameters()
aflocal.init()
summary = aflocal.run(writeflags=True)
aflocal.done()
self.assertEqual(summary['report0']['flagged'], 11175.0)
self.assertEqual(summary['report0']['total'], 1248000)
self.assertEqual(summary['report0']['correlation']['Sol1']['flagged'], 11136.0)
self.assertEqual(summary['report0']['correlation']['Sol2']['flagged'], 39)
def test_manual_antenna_selection_agent_layer_for_tsys_CalTable2(self):
"""AgentFlagger:: Manually flag a Tsys-based CalTable using flag agent selection engine for antenna"""
# Run the previous test but using the specific parsing functions
aflocal = aftool()
aflocal.open(self.vis)
aflocal.selectdata()
aflocal.parsemanualparameters(apply=False)
aflocal.parsemanualparameters(antenna='DV09')
aflocal.parsesummaryparameters()
aflocal.init()
summary = aflocal.run(writeflags=True)
aflocal.done()
self.assertEqual(summary['report0']['antenna']['DV09']['flagged'], 14336)
self.assertEqual(summary['report0']['antenna']['DV10']['flagged'], 0)
def test_display_data_single_channel_selection(self):
"""AgentFlagger:: Check nominal behaviour for single spw:chan selection """
self.setUp_4Ants()
aflocal = aftool()
aflocal.open(self.vis)
aflocal.selectdata()
agentUnflag={'apply':True,'mode':'unflag'}
agentManual={'apply':True,'mode':'manual','spw':'*:20~40'}
agentSummary={'apply':True,'mode':'summary'}
agentDisplay={'mode':'display','datadisplay':True,'pause':False}
aflocal.parseagentparameters(agentManual)
aflocal.parseagentparameters(agentSummary)
aflocal.parseagentparameters(agentDisplay)
aflocal.init()
summary = aflocal.run(writeflags=True)
aflocal.done()
def test_manual_antenna_selection_agent_layer_for_tsys_CalTable2(self):
"""AgentFlagger:: Manually flag a Tsys-based CalTable using flag agent selection engine for antenna"""
# Run the previous test but using the specific parsing functions
aflocal = aftool()
aflocal.open(self.vis)
aflocal.selectdata()
aflocal.parsemanualparameters(apply=False)
aflocal.parsemanualparameters(antenna='DV09')
aflocal.parsesummaryparameters()
aflocal.init()
summary = aflocal.run(writeflags=True)
aflocal.done()
self.assertEqual(summary['report0']['antenna']['DV09']['flagged'], 14336)
self.assertEqual(summary['report0']['antenna']['DV10']['flagged'], 0)
def test_display_data_single_channel_selection(self):
"""AgentFlagger:: Check nominal behaviour for single spw:chan selection """
self.setUp_4Ants()
aflocal = aftool()
aflocal.open(self.vis)
aflocal.selectdata()
agentUnflag = {'apply': True, 'mode': 'unflag'}
agentManual = {'apply': True, 'mode': 'manual', 'spw': '*:20~40'}
agentSummary = {'apply': True, 'mode': 'summary'}
agentDisplay = {'mode': 'display', 'datadisplay': True, 'pause': False}
aflocal.parseagentparameters(agentManual)
aflocal.parseagentparameters(agentSummary)
aflocal.parseagentparameters(agentDisplay)
aflocal.init()
summary = aflocal.run(writeflags=True)
aflocal.done()
def test_null_intent_selection1(self):
'''Agentflagger: handle unknown scan intent in list mode'''
aflocal = aftool()
aflocal.open(self.vis)
aflocal.selectdata()
aflocal.parsemanualparameters(intent='FOCUS') # non-existing intent
aflocal.parsemanualparameters(intent='CALIBRATE_POINTING*') # scan=1
aflocal.parsemanualparameters(intent='CALIBRATE_AMPLI_ON_SOURC') # typo
aflocal.parsemanualparameters(intent='*DELAY*') # non-existing
aflocal.parsesummaryparameters()
aflocal.init()
summary = aflocal.run()
aflocal.done()
self.assertEqual(summary['report0']['scan']['1']['flagged'], 192416)
self.assertEqual(summary['report0']['scan']['8']['flagged'], 0)
self.assertEqual(summary['report0']['flagged'], 192416)
def test_display_data_different_corrs_per_spw(self):
"""AgentFlagger:: Check behaviour when the number of correlation products changes between SPWs """
self.setUp_CAS_4052()
aflocal = aftool()
aflocal.open(self.vis)
aflocal.selectdata()
agentUnflag={'apply':True,'mode':'unflag'}
agentManual={'apply':True,'mode':'manual','spw':'*:100~200;300~400'}
agentSummary={'apply':True,'mode':'summary'}
agentDisplay={'mode':'display','datadisplay':True,'pause':False}
aflocal.parseagentparameters(agentUnflag)
aflocal.parseagentparameters(agentManual)
aflocal.parseagentparameters(agentSummary)
aflocal.parseagentparameters(agentDisplay)
aflocal.init()
summary = aflocal.run(writeflags=True)
aflocal.done()
def test_null_intent_selection1(self):
'''Agentflagger: handle unknown scan intent in list mode'''
aflocal = aftool()
aflocal.open(self.vis)
aflocal.selectdata()
aflocal.parsemanualparameters(intent='FOCUS') # non-existing intent
aflocal.parsemanualparameters(intent='CALIBRATE_POINTING*') # scan=1
aflocal.parsemanualparameters(intent='CALIBRATE_AMPLI_ON_SOURC') # typo
aflocal.parsemanualparameters(intent='*DELAY*') # non-existing
aflocal.parsesummaryparameters()
aflocal.init()
summary = aflocal.run()
aflocal.done()
self.assertEqual(summary['report0']['scan']['1']['flagged'], 192416)
self.assertEqual(summary['report0']['scan']['8']['flagged'], 0)
self.assertEqual(summary['report0']['flagged'], 192416)
def test_rflag_cparam_sol2(self):
"""AgentFlagger:: Test rflag solution 2 of CPARAM column for bpass"""
aflocal = aftool()
correlation = 'Sol2'
aflocal.open(self.vis)
aflocal.selectdata()
agentRflag={'apply':True,'mode':'rflag','correlation':correlation,
'extendflags':False,'datacolumn':'CPARAM'}
agentSummary={'apply':True,'mode':'summary'}
aflocal.parseagentparameters(agentRflag)
aflocal.parseagentparameters(agentSummary)
aflocal.init()
summary = aflocal.run(writeflags=True)
aflocal.done()
self.assertEqual(summary['report0']['flagged'], 13197)
self.assertEqual(summary['report0']['correlation']['Sol1']['flagged'], 0)
self.assertEqual(summary['report0']['correlation']['Sol2']['flagged'], 13197)
def test_display_data_different_corrs_per_spw(self):
"""AgentFlagger:: Check behaviour when the number of correlation products changes between SPWs """
self.setUp_CAS_4052()
aflocal = aftool()
aflocal.open(self.vis)
aflocal.selectdata()
agentUnflag={'apply':True,'mode':'unflag'}
agentManual={'apply':True,'mode':'manual','spw':'*:100~200;300~400'}
agentSummary={'apply':True,'mode':'summary'}
agentDisplay={'mode':'display','datadisplay':True,'pause':False}
aflocal.parseagentparameters(agentUnflag)
aflocal.parseagentparameters(agentManual)
aflocal.parseagentparameters(agentSummary)
aflocal.parseagentparameters(agentDisplay)
aflocal.init()
summary = aflocal.run(writeflags=True)
aflocal.done()
def test_rflag_cparam_sol2(self):
"""AgentFlagger:: Test rflag solution 2 of CPARAM column for bpass"""
aflocal = aftool()
correlation = 'Sol2'
aflocal.open(self.vis)
aflocal.selectdata()
agentRflag={'apply':True,'mode':'rflag','correlation':correlation,
'extendflags':False,'datacolumn':'CPARAM'}
agentSummary={'apply':True,'mode':'summary'}
aflocal.parseagentparameters(agentRflag)
aflocal.parseagentparameters(agentSummary)
aflocal.init()
summary = aflocal.run(writeflags=True)
aflocal.done()
self.assertEqual(summary['report0']['flagged'], 13197)
self.assertEqual(summary['report0']['correlation']['Sol1']['flagged'], 0)
self.assertEqual(summary['report0']['correlation']['Sol2']['flagged'], 13197)
def test_manual_antenna_selection_agent_layer_for_tsys_CalTable(self):
"""AgentFlagger:: Manually flag a Tsys-based CalTable using flag agent selection engine for antenna"""
aflocal = aftool()
aflocal.open(self.vis)
aflocal.selectdata()
agentUnflag={'apply':True,'mode':'unflag'}
agentManual={'apply':True,'mode':'manual','antenna':'DV09'}
agentSummary={'apply':True,'mode':'summary'}
aflocal.parseagentparameters(agentUnflag)
aflocal.parseagentparameters(agentManual)
aflocal.parseagentparameters(agentSummary)
aflocal.init()
summary = aflocal.run(writeflags=True)
aflocal.done()
self.assertEqual(summary['report0']['antenna']['DV09']['flagged'], 14336)
self.assertEqual(summary['report0']['antenna']['DV10']['flagged'], 0)
def test_manual_antenna_selection_agent_layer_for_bpass_CalTable(self):
"""AgentFlagger:: Manually flag a bpass-based CalTable using flag agent selection engine for antenna"""
aflocal = aftool()
aflocal.open(self.vis)
aflocal.selectdata()
agentUnflag={'apply':True,'mode':'unflag'}
agentManual={'apply':True,'mode':'manual','antenna':'ea09'}
agentSummary={'apply':True,'mode':'summary'}
aflocal.parseagentparameters(agentUnflag)
aflocal.parseagentparameters(agentManual)
aflocal.parseagentparameters(agentSummary)
aflocal.init()
summary = aflocal.run(writeflags=True)
aflocal.done()
self.assertEqual(summary['report0']['antenna']['ea09']['flagged'], 48000.0)
self.assertEqual(summary['report0']['antenna']['ea10']['flagged'], 0.0)
def test_clip_minmax_fparm_sol1_extension(self):
"""AgentFlagger:: Test cliping first calibration solution product of FPARAM column using a minmax range, and then extend to the other solution """
aflocal = aftool()
datacolumn = 'FPARAM'
aflocal.open(self.vis)
aflocal.selectdata()
agentUnflag={'apply':True,'mode':'unflag'}
agentClip={'apply':True,'mode':'clip','clipzeros':True,'clipminmax':[0.,600.],'datacolumn':datacolumn,'correlation':'Sol1'}
agentExtension={'apply':True,'mode':'extend','extendpols':True,'growfreq':0.0, 'growtime':0.0}
agentSummary={'apply':True,'mode':'summary','correlation':'""'}
aflocal.parseagentparameters(agentUnflag)
aflocal.parseagentparameters(agentClip)
aflocal.parseagentparameters(agentExtension)
aflocal.parseagentparameters(agentSummary)
aflocal.init()
summary = aflocal.run(writeflags=True)
aflocal.done()
self.assertEqual(summary['report0']['total'], 129024)
self.assertEqual(summary['report0']['flagged'], 1500)
self.assertEqual(summary['report0']['correlation']['Sol1']['flagged'], 750)
self.assertEqual(summary['report0']['correlation']['Sol2']['flagged'], 750)
def test_clip_minmax_fparm_sol1_2(self):
"""AgentFlagger:: Test cliping first calibration solution product of FPARAM column using a minmax range """
# Run the previous test but using the specific parsing functions
aflocal = aftool()
datacolumn = 'FPARAM'
correlation = 'Sol1'
aflocal.open(self.vis)
aflocal.selectdata()
aflocal.parsemanualparameters(apply=False)
aflocal.parseclipparameters(clipzeros=True,clipminmax=[0.,600.],datacolumn=datacolumn,
correlation=correlation)
aflocal.parsesummaryparameters()
aflocal.init()
summary = aflocal.run(writeflags=True)
aflocal.done()
self.assertEqual(summary['report0']['total'], 129024)
self.assertEqual(summary['report0']['flagged'], 750)
self.assertEqual(summary['report0']['correlation']['Sol1']['flagged'], 750)
self.assertEqual(summary['report0']['correlation']['Sol1']['total'], 64512)
self.assertEqual(summary['report0']['correlation']['Sol2']['flagged'], 0)
self.assertEqual(summary['report0']['correlation']['Sol2']['total'], 64512)
def test_clip_minmax_fparm_sol1_extension(self):
"""AgentFlagger:: Test cliping first calibration solution product of FPARAM column using a minmax range, and then extend to the other solution """
aflocal = aftool()
datacolumn = 'FPARAM'
aflocal.open(self.vis)
aflocal.selectdata()
agentUnflag={'apply':True,'mode':'unflag'}
agentClip={'apply':True,'mode':'clip','clipzeros':True,'clipminmax':[0.,600.],'datacolumn':datacolumn,'correlation':'Sol1'}
agentExtension={'apply':True,'mode':'extend','extendpols':True,'growfreq':0.0, 'growtime':0.0}
agentSummary={'apply':True,'mode':'summary','correlation':'""'}
aflocal.parseagentparameters(agentUnflag)
aflocal.parseagentparameters(agentClip)
aflocal.parseagentparameters(agentExtension)
aflocal.parseagentparameters(agentSummary)
aflocal.init()
summary = aflocal.run(writeflags=True)
aflocal.done()
self.assertEqual(summary['report0']['total'], 129024)
self.assertEqual(summary['report0']['flagged'], 1500)
self.assertEqual(summary['report0']['correlation']['Sol1']['flagged'], 750)
self.assertEqual(summary['report0']['correlation']['Sol2']['flagged'], 750)
def test_summarylist1(self):
'''agentflagger: multiple summaries'''
aflocal = aftool()
aflocal.open(self.vis)
aflocal.selectdata()
aflocal.parsesummaryparameters(name='summary_1')
aflocal.parsemanualparameters(intent='FOCUS') # non-existing intent
aflocal.parsesummaryparameters(name='summary_2')
aflocal.parsemanualparameters(spw='0')
aflocal.parsesummaryparameters(name='summary_3')
aflocal.init()
summary = aflocal.run()
aflocal.done()
# It creates 2 reports per summary agent. One is type='summary',
# ther other is type='plotpoints'
self.assertEqual(summary['report0']['name'],'summary_1')
self.assertEqual(summary['report2']['name'],'summary_2')
self.assertEqual(summary['report4']['name'],'summary_3')
self.assertEqual(summary['report0']['flagged'], 0)
self.assertEqual(summary['report2']['flagged'], 0)
self.assertEqual(summary['report4']['spw']['0']['flagged'], summary['report4']['spw']['0']['total'])
def test_clip_minmax_fparm_all(self):
"""AgentFlagger:: Test cliping all calibration solution products of FPARAM column using a minmax range """
aflocal = aftool()
datacolumn = 'FPARAM'
correlation = ''
aflocal.open(self.vis)
aflocal.selectdata()
agentUnflag={'apply':True,'mode':'unflag'}
agentClip={'apply':True,'mode':'clip','clipzeros':True,'clipminmax':[0.,600.],'datacolumn':datacolumn,'correlation':correlation}
agentSummary={'apply':True,'mode':'summary'}
aflocal.parseagentparameters(agentUnflag)
aflocal.parseagentparameters(agentClip)
aflocal.parseagentparameters(agentSummary)
aflocal.init()
summary = aflocal.run(writeflags=True)
aflocal.done()
self.assertEqual(summary['report0']['total'], 129024.0)
self.assertEqual(summary['report0']['flagged'], 1192.0)
self.assertEqual(summary['report0']['correlation']['Sol1']['flagged'], 750.0)
self.assertEqual(summary['report0']['correlation']['Sol1']['total'], 64512.0)
self.assertEqual(summary['report0']['correlation']['Sol2']['flagged'], 442.0)
self.assertEqual(summary['report0']['correlation']['Sol2']['total'], 64512.0)
def test_clip_nan_and_inf_fparm_all(self):
"""AgentFlagger:: Test cliping only NaNs/Infs in all calibration solution products of FPARAM column"""
aflocal = aftool()
datacolumn = 'FPARAM'
correlation = ''
aflocal.open(self.vis)
aflocal.selectdata()
agentUnflag={'apply':True,'mode':'unflag'}
agentClip={'apply':True,'mode':'clip','datacolumn':datacolumn,'correlation':correlation}
agentSummary={'apply':True,'mode':'summary'}
aflocal.parseagentparameters(agentUnflag)
aflocal.parseagentparameters(agentClip)
aflocal.parseagentparameters(agentSummary)
aflocal.init()
summary = aflocal.run(writeflags=True)
aflocal.done()
self.assertEqual(summary['report0']['total'], 129024.0)
self.assertEqual(summary['report0']['flagged'], 0.0)
self.assertEqual(summary['report0']['correlation']['Sol1']['flagged'], 0.0)
self.assertEqual(summary['report0']['correlation']['Sol1']['total'], 64512.0)
self.assertEqual(summary['report0']['correlation']['Sol2']['flagged'], 0.0)
self.assertEqual(summary['report0']['correlation']['Sol2']['total'], 64512.0)
def test_clip_minmax_fparm_error_case_absall(self):
"""AgentFlagger:: Error case test when a complex operator is used with CalTables """
aflocal = aftool()
datacolumn = 'FPARAM'
correlation = 'ABS_ALL'
aflocal.open(self.vis)
aflocal.selectdata()
agentUnflag={'apply':True,'mode':'unflag'}
agentClip={'apply':True,'mode':'clip','clipzeros':True,'clipminmax':[0.,600.],'datacolumn':datacolumn,'correlation':correlation}
agentSummary={'apply':True,'mode':'summary'}
aflocal.parseagentparameters(agentUnflag)
aflocal.parseagentparameters(agentClip)
aflocal.parseagentparameters(agentSummary)
aflocal.init()
summary = aflocal.run(writeflags=True)
aflocal.done()
self.assertEqual(summary['report0']['total'], 129024.0)
self.assertEqual(summary['report0']['flagged'], 1192.0)
self.assertEqual(summary['report0']['correlation']['Sol1']['flagged'], 750.0)
self.assertEqual(summary['report0']['correlation']['Sol1']['total'], 64512.0)
self.assertEqual(summary['report0']['correlation']['Sol2']['flagged'], 442.0)
self.assertEqual(summary['report0']['correlation']['Sol2']['total'], 64512.0)
def test_clip_minmax_snr_all_for_bpass_CalTable(self):
"""AgentFlagger:: Test cliping all calibration solution products of SNR column using a minmax range for bpass CalTable"""
aflocal = aftool()
datacolumn = 'SNR'
correlation = ''
aflocal.open(self.vis)
aflocal.selectdata()
agentUnflag={'apply':True,'mode':'unflag'}
agentClip={'apply':True,'mode':'clip','clipzeros':True,'clipminmax':[0.,550.],'datacolumn':datacolumn,'correlation':correlation}
agentSummary={'apply':True,'mode':'summary'}
aflocal.parseagentparameters(agentUnflag)
aflocal.parseagentparameters(agentClip)
aflocal.parseagentparameters(agentSummary)
aflocal.init()
summary = aflocal.run(writeflags=True)
aflocal.done()
self.assertEqual(summary['report0']['total'], 1248000.0)
self.assertEqual(summary['report0']['flagged'], 74371.0)
self.assertEqual(summary['report0']['correlation']['Sol1']['flagged'], 36327.0)
self.assertEqual(summary['report0']['correlation']['Sol1']['total'], 624000.0)
self.assertEqual(summary['report0']['correlation']['Sol2']['flagged'], 38044.0)
self.assertEqual(summary['report0']['correlation']['Sol2']['total'], 624000.0)
def test_manual_antenna_msSelection_layer_for_tsys_CalTable(self):
"""AgentFlagger:: Manually flag a Tsys-based CalTable using MSSelection for antenna"""
aflocal = aftool()
aflocal.open(self.vis)
aflocal.selectdata(antenna='DV09')
agentUnflag={'apply':True,'mode':'unflag'}
agentManual={'apply':True,'mode':'manual'}
aflocal.parseagentparameters(agentUnflag)
aflocal.parseagentparameters(agentManual)
aflocal.init()
summary = aflocal.run(writeflags=True)
aflocal.done()
aflocal.open(self.vis)
aflocal.selectdata()
agentSummary={'apply':True,'mode':'summary'}
aflocal.parseagentparameters(agentSummary)
aflocal.init()
summary = aflocal.run(writeflags=True)
aflocal.done()
self.assertEqual(summary['report0']['antenna']['DV09']['flagged'], 14336)
self.assertEqual(summary['report0']['antenna']['DV10']['flagged'], 0)
os.system('rm -f '+ inp)
# save to a file
with open(inp, 'w') as f:
f.write(cmd)
f.close()
return
# Path for data
datapath = os.environ.get('CASAPATH').split()[0] + "/data/regression/unittest/flagdata/"
# Local copy of the agentflagger tool
aflocal = aftool()
# Base class which defines setUp functions
# for importing different data sets
class test_base(unittest.TestCase):
def setUp_ngc5921(self):
self.vis = "ngc5921.ms"
if os.path.exists(self.vis):
print "The MS is already around, just unflag"
else:
os.system('cp -r '+datapath + self.vis +' '+ self.vis)
os.system('rm -rf ' + self.vis + '.flagversions')
self.unflag_ms()
# Pick up alternative data directory to run tests on MMSs
testmms = False
if os.environ.has_key('TEST_DATADIR'):
DATADIR = str(os.environ.get('TEST_DATADIR'))+'/flagdata/'
if os.path.isdir(DATADIR):
testmms = True
datapath = DATADIR
print 'flagmanager tests will use data from '+datapath
# jagonzal (CAS-4287): Add a cluster-less mode to by-pass parallel processing for MMSs as requested
if os.environ.has_key('BYPASS_PARALLEL_PROCESSING'):
ParallelTaskHelper.bypassParallelProcessing(1)
# Local copy of the agentflagger tool
aflocal = aftool()
# Base class which defines setUp functions for importing different data sets
class test_base(unittest.TestCase):
def setUp_flagdatatest(self):
'''VLA data set, scan=2500~2600 spw=0 1 chan, RR,LL'''
self.vis = "flagdatatest.ms"
if testmms:
self.vis = "flagdatatest.mms"
if os.path.exists(self.vis):
print "The MS is already around, just unflag"
else:
print "Moving data..."
os.system('cp -r '+datapath + self.vis +' '+ self.vis)
def flagdata(
vis,
mode,
autocorr, # mode manual parameter
inpfile, # mode list parameters
reason,
tbuff,
spw, # data selection parameters
field,
antenna,
uvrange,
timerange,
correlation,
scan,
intent,
array,
observation,
feed,
clipminmax, # mode clip parameters
datacolumn,
clipoutside,
channelavg,
chanbin,
timeavg,
timebin,
clipzeros,
quackinterval, # mode quack parameters
quackmode,
quackincrement,
tolerance, # mode shadow parameter
addantenna,
lowerlimit, # mode elevation parameters
upperlimit,
ntime, # mode tfcrop
combinescans,
timecutoff,
freqcutoff,
timefit,
freqfit,
maxnpieces,
flagdimension,
usewindowstats,
halfwin,
extendflags,
winsize, # rflag parameters
timedev,
freqdev,
timedevscale,
freqdevscale,
spectralmax,
spectralmin,
antint_ref_antenna, # mode antint
minchanfrac,
verbose,
extendpols, # mode extend
growtime,
growfreq,
growaround,
flagneartime,
flagnearfreq,
minrel, # mode summary
maxrel,
minabs,
maxabs,
spwchan,
spwcorr,
basecnt,
fieldcnt,
name,
action, # run or not the tool
display,
flagbackup,
savepars, # save the current parameters to FLAG_CMD or to a file
cmdreason, # reason to save to flag cmd
outfile, # output file to save flag commands
overwrite, # overwrite the outfile file
writeflags # HIDDEN parameter
):
#
# Task flagdata
# Flags data from an MS or calibration table based on data selection in various ways
casalog.origin('flagdata')
if (action == 'none' or action == '' or action == 'calculate'):
flagbackup = False
# SMC: moved the flagbackup to before initializing the cluster.
# Note that with this change, a flag backup will be created even if
# an error happens that prevents the flagger tool from running.
if (mode != 'summary' and flagbackup):
casalog.post('Backup original flags before applying new flags')
fh.backupFlags(aflocal=None, msfile=vis, prename='flagdata')
# Set flagbackup to False because only the controller
# should create a backup
flagbackup = False
# Initialize the helper class
orig_locals = copy.deepcopy(locals())
FHelper = FlagHelper()
# Check if vis is a MS, MMS or cal table:
# typevis = 1 --> cal table
# typevis = 0 --> MS
# typevis = 2 --> MMS
iscal = False
typevis = fh.isCalTable(vis)
if typevis == 1:
iscal = True
# ***************** Input is MMS -- Parallel Processing ***********************
if FHelper.isMPIEnabled(
) and typevis == 2 and action != '' and action != 'none':
# Create a temporary input file with .tmp extension.
# Use this file for all the processing from now on.
if (isinstance(inpfile,str) and inpfile != '') or \
(isinstance(inpfile, list) and os.path.isfile(inpfile[0])):
inpfile = FHelper.setupInputFile(inpfile)
if inpfile != None:
orig_locals['inpfile'] = inpfile
if outfile != '':
outfile = os.path.abspath(outfile)
orig_locals['outfile'] = outfile
if isinstance(addantenna, str) and addantenna != '':
addantenna = os.path.abspath(addantenna)
orig_locals['addantenna'] = addantenna
if isinstance(timedev, str) and timedev != '':
timedev = os.path.abspath(timedev)
orig_locals['timedev'] = timedev
if isinstance(freqdev, str) and freqdev != '':
freqdev = os.path.abspath(freqdev)
orig_locals['freqdev'] = freqdev
FHelper.__init__(orig_locals)
# For tests only
# FHelper.bypassParallelProcessing(1)
FHelper.setupCluster('flagdata')
# (CAS-4119): Override summary minabs,maxabs,minrel,maxrel
# so that it is done after consolidating the summaries
# By-pass options to filter summary
filterSummary = False
if ((mode == 'summary') and ((minrel != 0.0) or (maxrel != 1.0) or
(minabs != 0) or (maxabs != -1))):
filterSummary = True
myms = mstool()
myms.open(vis)
subMS_list = myms.getreferencedtables()
myms.close()
if (minrel != 0.0):
minreal_dict = create_arg_dict(subMS_list, 0.0)
FHelper.override_arg('minrel', minreal_dict)
if (maxrel != 1.0):
maxrel_dict = create_arg_dict(subMS_list, 1.0)
FHelper.override_arg('maxrel', maxrel_dict)
if (minabs != 0):
minabs_dict = create_arg_dict(subMS_list, 0)
FHelper.override_arg('minabs', minabs_dict)
if (maxabs != -1):
maxabs_dict = create_arg_dict(subMS_list, -1)
FHelper.override_arg('maxabs', maxabs_dict)
# By-pass options to filter summary
if savepars:
myms = mstool()
myms.open(vis)
subMS_list = myms.getreferencedtables()
myms.close()
savepars_dict = create_arg_dict(subMS_list, False)
FHelper.override_arg('savepars', savepars_dict)
# Execute the parallel engines
retVar = FHelper.go()
# Save overall RFlag values into timedev/freqdev files once the subMS results have
# been consolidated, if needed for RFlag (introduced in CAS-11850)
opts_dict = {
'writeflags': writeflags,
'timedev': timedev,
'freqdev': freqdev
}
fh.save_rflag_consolidated_files(mode, action, retVar, opts_dict,
inpfile)
# In async mode return the job ids
if ParallelTaskHelper.getAsyncMode():
return retVar
else:
# Filter summary at MMS level
if (mode == 'summary'):
if filterSummary:
retVar = filter_summary(retVar, minrel, maxrel, minabs,
maxabs)
return retVar
# Save parameters at MMS level
elif savepars:
action = 'none'
else:
return retVar
summary_stats = {}
# ***************** Input is a normal MS/cal table ****************
# Create local tools
# aflocal = casac.agentflagger()
aflocal = aftool()
mslocal = mstool()
try:
# Verify the ntime value
newtime = 0.0
if type(ntime) == float or type(ntime) == int:
if ntime <= 0:
raise Exception, 'Parameter ntime cannot be < = 0'
else:
# units are seconds
newtime = float(ntime)
elif type(ntime) == str:
if ntime == 'scan':
# iteration time step is a scan
newtime = 0.0
else:
# read the units from the string
qtime = qa.quantity(ntime)
if qtime['unit'] == 'min':
# convert to seconds
qtime = qa.convert(qtime, 's')
elif qtime['unit'] == '':
qtime['unit'] = 's'
# check units
if qtime['unit'] == 's':
newtime = qtime['value']
else:
casalog.post(
'Cannot convert units of ntime. Will use default 0.0s',
'WARN')
casalog.post(
"New ntime is of type %s and value %s" % (type(newtime), newtime),
'DEBUG')
# Open the MS and attach it to the tool
if ((type(vis) == str) & (os.path.exists(vis))):
aflocal.open(vis, newtime)
else:
raise Exception, 'Visibility data set not found - please verify the name'
# Get the parameters for the mode
agent_pars = {}
# By default, write flags to the MS
writeflags = True
# Only the apply action writes to the MS
# action=apply --> write to the MS
# action=calculate --> do not write to the MS
# action='' --> do not run the tool and do not write to the MS
if action != 'apply':
writeflags = False
# Default mode
if mode == '' or mode == 'manualflag':
mode = 'manual'
# Read in the list of commands
# Make a dictionary of the input commands. Select by reason if requested
flagcmd = {}
if mode == 'list':
casalog.post('List mode is active')
doPadding = True
try:
# If tbuff is requested, read and Parse
if tbuff == 0.0 or tbuff == [] or tbuff == None:
doPadding = False
if doPadding:
casalog.post('Will apply time buffer padding')
# inpfile is a file
if isinstance(inpfile, str):
inpfile = [inpfile]
# read in the list and do a simple parsing to apply tbuff
flaglist = fh.readAndParse(inpfile, tbuff)
else:
flaglist = fh.get_flag_cmd_list(inpfile)
# Parse and create a dictionary
flagcmd = fh.parseDictionary(flaglist, reason)
# Validate the dictionary.
# IMPORTANT: if any parameter changes its type, the following
# function needs to be updated. The same if any new parameter is
# added or removed from the task
fh.evaluateFlagParameters(flagcmd, orig_locals)
# List of flag commands in dictionary
vrows = flagcmd.keys()
casalog.post('%s' % flagcmd, 'DEBUG1')
except Exception, instance:
casalog.post('%s' % instance, 'ERROR')
raise Exception, 'Error reading the input list. Make sure the syntax used in the list '\
'follows the rules given in the inline help of the task.'
casalog.post('Selected ' + str(vrows.__len__()) +
' commands from combined input list(s) ')
elif mode == 'manual':
agent_pars['autocorr'] = autocorr
casalog.post('Manual mode is active')
