Exemplo n.º 1
0
    def test_flagversions(self):
        '''Partition: check that the .flagversions is created'''

        # Run partition and create the .flagversions
        partition(vis=self.msfile,
                  outputvis=self.mmsfile,
                  createmms=True,
                  disableparallel=True,
                  datacolumn='data')
        self.assertTrue(os.path.exists(self.mmsfile + '.flagversions'))

        # Check that the number of backups in MMS is correct
        aflocal = aftool()
        aflocal.open(self.mmsfile)
        nv = aflocal.getflagversionlist()
        aflocal.done()
        self.assertEqual(len(nv), 3)

        # Run flagdata on MMS to check if it works well.
        flagdata(vis=self.mmsfile, mode='unflag', flagbackup=True)

        # Check that the number of backups in MMS is correct
        aflocal = aftool()
        aflocal.open(self.mmsfile)
        nvref = aflocal.getflagversionlist()
        aflocal.done()
        self.assertEqual(len(nvref), 4)
Exemplo n.º 2
0
    def test_manual_field_msSelection_layer_for_tsys_CalTable(self):
        """AgentFlagger:: Manually flag a Tsys-based CalTable using MSSelection for field """
        aflocal = aftool()

        aflocal.open(self.vis)
        aflocal.selectdata(field='0')
        agentUnflag = {'apply': True, 'mode': 'unflag'}
        agentManual = {'apply': True, 'mode': 'manual'}
        aflocal.parseagentparameters(agentUnflag)
        aflocal.parseagentparameters(agentManual)
        aflocal.init()
        summary = aflocal.run(writeflags=True)
        aflocal.done()

        aflocal.open(self.vis)
        aflocal.selectdata()
        agentSummary = {'apply': True, 'mode': 'summary'}
        aflocal.parseagentparameters(agentSummary)
        aflocal.init()
        summary = aflocal.run(writeflags=True)
        aflocal.done()

        self.assertEqual(summary['report0']['field']['3c279']['flagged'],
                         9216.0)
        self.assertEqual(summary['report0']['field']['Titan']['flagged'], 0)
        self.assertEqual(summary['report0']['field']['TW Hya']['flagged'], 0)
        self.assertEqual(
            summary['report0']['field']['J1037-295=QSO']['flagged'], 0)
Exemplo n.º 3
0
    def test_clip_nan_and_inf_fparm_all(self):
        """AgentFlagger:: Test cliping only NaNs/Infs in all calibration solution products of FPARAM column"""
        aflocal = aftool()
        datacolumn = 'FPARAM'
        correlation = ''
        aflocal.open(self.vis)
        aflocal.selectdata()
        agentUnflag = {'apply': True, 'mode': 'unflag'}
        agentClip = {
            'apply': True,
            'mode': 'clip',
            'datacolumn': datacolumn,
            'correlation': correlation
        }
        agentSummary = {'apply': True, 'mode': 'summary'}
        aflocal.parseagentparameters(agentUnflag)
        aflocal.parseagentparameters(agentClip)
        aflocal.parseagentparameters(agentSummary)
        aflocal.init()
        summary = aflocal.run(writeflags=True)
        aflocal.done()

        self.assertEqual(summary['report0']['total'], 129024.0)
        self.assertEqual(summary['report0']['flagged'], 0.0)
        self.assertEqual(summary['report0']['correlation']['Sol1']['flagged'],
                         0.0)
        self.assertEqual(summary['report0']['correlation']['Sol1']['total'],
                         64512.0)
        self.assertEqual(summary['report0']['correlation']['Sol2']['flagged'],
                         0.0)
        self.assertEqual(summary['report0']['correlation']['Sol2']['total'],
                         64512.0)
Exemplo n.º 4
0
    def test_clip_minmax_fparm_all(self):
        """AgentFlagger:: Test cliping all calibration solution products of FPARAM column using a minmax range """
        aflocal = aftool()
        datacolumn = 'FPARAM'
        correlation = ''
        aflocal.open(self.vis)
        aflocal.selectdata()
        agentUnflag = {'apply': True, 'mode': 'unflag'}
        agentClip = {
            'apply': True,
            'mode': 'clip',
            'clipzeros': True,
            'clipminmax': [0., 600.],
            'datacolumn': datacolumn,
            'correlation': correlation
        }
        agentSummary = {'apply': True, 'mode': 'summary'}
        aflocal.parseagentparameters(agentUnflag)
        aflocal.parseagentparameters(agentClip)
        aflocal.parseagentparameters(agentSummary)
        aflocal.init()
        summary = aflocal.run(writeflags=True)
        aflocal.done()

        self.assertEqual(summary['report0']['total'], 129024.0)
        self.assertEqual(summary['report0']['flagged'], 1192.0)
        self.assertEqual(summary['report0']['correlation']['Sol1']['flagged'],
                         750.0)
        self.assertEqual(summary['report0']['correlation']['Sol1']['total'],
                         64512.0)
        self.assertEqual(summary['report0']['correlation']['Sol2']['flagged'],
                         442.0)
        self.assertEqual(summary['report0']['correlation']['Sol2']['total'],
                         64512.0)
Exemplo n.º 5
0
    def test_manual_field_msSelection_layer_for_bpass_CalTable(self):
        """AgentFlagger:: Manually flag a bpass-based CalTable using MSSelection for field """
        aflocal = aftool()

        aflocal.open(self.vis)
        aflocal.selectdata({'field': '3C286_A'})
        agentUnflag = {'apply': True, 'mode': 'unflag'}
        agentManual = {'apply': True, 'mode': 'manual'}
        aflocal.parseagentparameters(agentUnflag)
        aflocal.parseagentparameters(agentManual)
        aflocal.init()
        summary = aflocal.run(writeflags=True)
        aflocal.done()

        aflocal.open(self.vis)
        aflocal.selectdata()
        agentSummary = {'apply': True, 'mode': 'summary'}
        aflocal.parseagentparameters(agentSummary)
        aflocal.init()
        summary = aflocal.run(writeflags=True)
        aflocal.done()

        self.assertEqual(summary['report0']['field']['3C286_A']['flagged'],
                         499200.0)
        self.assertEqual(summary['report0']['field']['3C286_B']['flagged'], 0)
        self.assertEqual(summary['report0']['field']['3C286_C']['flagged'], 0)
        self.assertEqual(summary['report0']['field']['3C286_D']['flagged'], 0)
Exemplo n.º 6
0
    def test_clip_minmax_fparm_error_case_absall(self):
        """AgentFlagger:: Error case test when a complex operator is used with CalTables """
        aflocal = aftool()
        datacolumn = 'FPARAM'
        correlation = 'ABS_ALL'
        aflocal.open(self.vis)
        aflocal.selectdata()
        agentUnflag = {'apply': True, 'mode': 'unflag'}
        agentClip = {
            'apply': True,
            'mode': 'clip',
            'clipzeros': True,
            'clipminmax': [0., 600.],
            'datacolumn': datacolumn,
            'correlation': correlation
        }
        agentSummary = {'apply': True, 'mode': 'summary'}
        aflocal.parseagentparameters(agentUnflag)
        aflocal.parseagentparameters(agentClip)
        aflocal.parseagentparameters(agentSummary)
        aflocal.init()
        summary = aflocal.run(writeflags=True)
        aflocal.done()

        self.assertEqual(summary['report0']['total'], 129024.0)
        self.assertEqual(summary['report0']['flagged'], 1192.0)
        self.assertEqual(summary['report0']['correlation']['Sol1']['flagged'],
                         750.0)
        self.assertEqual(summary['report0']['correlation']['Sol1']['total'],
                         64512.0)
        self.assertEqual(summary['report0']['correlation']['Sol2']['flagged'],
                         442.0)
        self.assertEqual(summary['report0']['correlation']['Sol2']['total'],
                         64512.0)
Exemplo n.º 7
0
    def test_manual_antenna_msSelection_layer_for_bpass_CalTable(self):
        """AgentFlagger:: Manually flag a bpass-based CalTable using MSSelection for antenna"""
        aflocal = aftool()

        aflocal.open(self.vis)
        aflocal.selectdata({'antenna': 'ea09'})
        agentUnflag = {'apply': True, 'mode': 'unflag'}
        agentManual = {'apply': True, 'mode': 'manual'}
        aflocal.parseagentparameters(agentUnflag)
        aflocal.parseagentparameters(agentManual)
        aflocal.init()
        summary = aflocal.run(writeflags=True)
        aflocal.done()

        aflocal.open(self.vis)
        aflocal.selectdata()
        agentSummary = {'apply': True, 'mode': 'summary'}
        aflocal.parseagentparameters(agentSummary)
        aflocal.init()
        summary = aflocal.run(writeflags=True)
        aflocal.done()

        self.assertEqual(summary['report0']['antenna']['ea09']['flagged'],
                         48000.0)
        self.assertEqual(summary['report0']['antenna']['ea10']['flagged'], 0.0)
Exemplo n.º 8
0
    def test_manual_field_msSelection_layer_for_tsys_CalTable(self):
        """AgentFlagger:: Manually flag a Tsys-based CalTable using MSSelection for field """
        aflocal = aftool()

        aflocal.open(self.vis)
        aflocal.selectdata(field='0')
        agentUnflag={'apply':True,'mode':'unflag'}
        agentManual={'apply':True,'mode':'manual'}
        aflocal.parseagentparameters(agentUnflag)
        aflocal.parseagentparameters(agentManual)
        aflocal.init()
        summary = aflocal.run(writeflags=True)
        aflocal.done() 

        aflocal.open(self.vis)
        aflocal.selectdata()
        agentSummary={'apply':True,'mode':'summary'}
        aflocal.parseagentparameters(agentSummary)
        aflocal.init()
        summary = aflocal.run(writeflags=True)
        aflocal.done() 

        self.assertEqual(summary['report0']['field']['3c279']['flagged'], 9216.0)
        self.assertEqual(summary['report0']['field']['Titan']['flagged'], 0)
        self.assertEqual(summary['report0']['field']['TW Hya']['flagged'], 0)
        self.assertEqual(summary['report0']['field']['J1037-295=QSO']['flagged'], 0)        
Exemplo n.º 9
0
    def test_clip_minmax_snr_all_for_bpass_CalTable(self):
        """AgentFlagger:: Test cliping all calibration solution products of SNR column using a minmax range for bpass CalTable"""
        aflocal = aftool()
        datacolumn = 'SNR'
        correlation = ''
        aflocal.open(self.vis)
        aflocal.selectdata()
        agentUnflag = {'apply': True, 'mode': 'unflag'}
        agentClip = {
            'apply': True,
            'mode': 'clip',
            'clipzeros': True,
            'clipminmax': [0., 550.],
            'datacolumn': datacolumn,
            'correlation': correlation
        }
        agentSummary = {'apply': True, 'mode': 'summary'}
        aflocal.parseagentparameters(agentUnflag)
        aflocal.parseagentparameters(agentClip)
        aflocal.parseagentparameters(agentSummary)
        aflocal.init()
        summary = aflocal.run(writeflags=True)
        aflocal.done()

        self.assertEqual(summary['report0']['total'], 1248000.0)
        self.assertEqual(summary['report0']['flagged'], 74371.0)
        self.assertEqual(summary['report0']['correlation']['Sol1']['flagged'],
                         36327.0)
        self.assertEqual(summary['report0']['correlation']['Sol1']['total'],
                         624000.0)
        self.assertEqual(summary['report0']['correlation']['Sol2']['flagged'],
                         38044.0)
        self.assertEqual(summary['report0']['correlation']['Sol2']['total'],
                         624000.0)
Exemplo n.º 10
0
    def test_clip_minmax_fparm_sol1_2(self):
        """AgentFlagger:: Test cliping first calibration solution product of FPARAM column using a minmax range """

        # Run the previous test but using the specific parsing functions
        aflocal = aftool()
        datacolumn = 'FPARAM'
        correlation = 'Sol1'
        aflocal.open(self.vis)
        aflocal.selectdata()
        aflocal.parsemanualparameters(apply=False)
        aflocal.parseclipparameters(clipzeros=True,
                                    clipminmax=[0., 600.],
                                    datacolumn=datacolumn,
                                    correlation=correlation)
        aflocal.parsesummaryparameters()
        aflocal.init()
        summary = aflocal.run(writeflags=True)
        aflocal.done()
        self.assertEqual(summary['report0']['total'], 129024)
        self.assertEqual(summary['report0']['flagged'], 750)
        self.assertEqual(summary['report0']['correlation']['Sol1']['flagged'],
                         750)
        self.assertEqual(summary['report0']['correlation']['Sol1']['total'],
                         64512)
        self.assertEqual(summary['report0']['correlation']['Sol2']['flagged'],
                         0)
        self.assertEqual(summary['report0']['correlation']['Sol2']['total'],
                         64512)
Exemplo n.º 11
0
    def test_manual_field_msSelection_layer_for_bpass_CalTable(self):
        """AgentFlagger:: Manually flag a bpass-based CalTable using MSSelection for field """
        aflocal = aftool()

        aflocal.open(self.vis)
        aflocal.selectdata({'field':'3C286_A'})
        agentUnflag={'apply':True,'mode':'unflag'}
        agentManual={'apply':True,'mode':'manual'}
        aflocal.parseagentparameters(agentUnflag)
        aflocal.parseagentparameters(agentManual)
        aflocal.init()
        summary = aflocal.run(writeflags=True)
        aflocal.done() 

        aflocal.open(self.vis)
        aflocal.selectdata()
        agentSummary={'apply':True,'mode':'summary'}
        aflocal.parseagentparameters(agentSummary)
        aflocal.init()
        summary = aflocal.run(writeflags=True)
        aflocal.done() 

        self.assertEqual(summary['report0']['field']['3C286_A']['flagged'], 499200.0)
        self.assertEqual(summary['report0']['field']['3C286_B']['flagged'], 0)
        self.assertEqual(summary['report0']['field']['3C286_C']['flagged'], 0)
        self.assertEqual(summary['report0']['field']['3C286_D']['flagged'], 0)
Exemplo n.º 12
0
    def test_summarylist1(self):
        '''agentflagger: multiple summaries'''
        aflocal = aftool()
        aflocal.open(self.vis)
        aflocal.selectdata()
        aflocal.parsesummaryparameters(name='summary_1')
        aflocal.parsemanualparameters(intent='FOCUS')  # non-existing intent
        aflocal.parsesummaryparameters(name='summary_2')
        aflocal.parsemanualparameters(spw='0')
        aflocal.parsesummaryparameters(name='summary_3')
        aflocal.init()
        summary = aflocal.run()
        aflocal.done()

        # It creates 2 reports per summary agent. One is type='summary',
        # the other is type='plotpoints'
        # jagonzal: Not anymore, now it creates only 1 report per summary
        # agent unless display is 'report' or 'both'
        self.assertEqual(summary['report0']['name'], 'summary_1')
        self.assertEqual(summary['report1']['name'], 'summary_2')
        self.assertEqual(summary['report2']['name'], 'summary_3')
        self.assertEqual(summary['report0']['flagged'], 0)
        self.assertEqual(summary['report1']['flagged'], 0)
        self.assertEqual(summary['report2']['spw']['0']['flagged'],
                         summary['report2']['spw']['0']['total'])
Exemplo n.º 13
0
    def test_display_cal_tables(self):
        '''AgentFlagger: Select spws, display and flag cal tables'''
        self.setUp_bpass_case()
        aflocal = aftool()
        aflocal.open(self.vis)
        aflocal.selectdata(spw='1')
        aflocal.parsemanualparameters(spw='1:0~10;60~63')
        aflocal.parsesummaryparameters(spwchan=True)
#        agentManual={'apply':True,'mode':'manual'}
#        agentSummary={'apply':True,'mode':'summary','spwchan':True}
        agentDisplay={'mode':'display','datadisplay':True,'pause':False,'datacolumn':'CPARAM'}
#        aflocal.parseagentparameters(agentManual)
#        aflocal.parseagentparameters(agentSummary)
        aflocal.parseagentparameters(agentDisplay)
        aflocal.init()
        summary = aflocal.run(writeflags=True)
        aflocal.done()
        
        self.assertEqual(summary['report0']['spw:channel']['1:0']['flagged'], 1300)
        self.assertEqual(summary['report0']['spw:channel']['1:5']['flagged'], 1300)
        self.assertEqual(summary['report0']['spw:channel']['1:10']['flagged'], 1300)
        self.assertEqual(summary['report0']['spw:channel']['1:11']['flagged'], 0)
        self.assertEqual(summary['report0']['spw:channel']['1:60']['flagged'], 1300)
        self.assertEqual(summary['report0']['spw:channel']['1:59']['flagged'], 0)
        self.assertEqual(summary['report0']['flagged'], 15*1300)
Exemplo n.º 14
0
 def test_tfcrop_cparam_sol1_extension(self):
     """AgentFlagger:: Test tfcrop first calibration solution product of CPARAM column, 
     and then extend to the other solution for bpass CalTable"""
     aflocal = aftool()
     datacolumn = "CPARAM"
     correlation = 'Sol1'
     aflocal.open(self.vis)
     aflocal.selectdata()
     
     # Pre-clip data to avoid problems with near-zero values
     agentClip={'apply':True,'mode':'clip','clipzeros':True,'datacolumn':datacolumn,'correlation':correlation}
     aflocal.parseagentparameters(agentClip)
     
     aflocal.parsetfcropparameters(datacolumn=datacolumn, correlation=correlation,
                                   extendflags=False)
     aflocal.parsesummaryparameters()
     # Extend to the other solution
     aflocal.parseextendparameters(extendpols=True,growfreq=0.0,growtime=0.0)
     aflocal.parsesummaryparameters()
     aflocal.init()
     summary = aflocal.run(writeflags=True)
     aflocal.done()
     
     # flags from first summary, only tfcrop     
     assert abs(summary['report0']['flagged'] - 30427) <= 5        
     assert abs(summary['report0']['correlation']['Sol1']['flagged'] - 30427) <= 5
     
     # flags from second summary, tfcrop+extend
     assert abs(summary['report1']['flagged'] - 2*30427) <= 10
     assert abs(summary['report1']['correlation']['Sol2']['flagged'] - 30427) <= 5        
Exemplo n.º 15
0
    def test_display_cal_tables(self):
        '''AgentFlagger: Select spws, display and flag cal tables'''
        self.setUp_bpass_case()
        aflocal = aftool()
        aflocal.open(self.vis)
        aflocal.selectdata(spw='1')
        aflocal.parsemanualparameters(spw='1:0~10;60~63')
        aflocal.parsesummaryparameters(spwchan=True)
#        agentManual={'apply':True,'mode':'manual'}
#        agentSummary={'apply':True,'mode':'summary','spwchan':True}
        agentDisplay={'mode':'display','datadisplay':True,'pause':False,'datacolumn':'CPARAM'}
#        aflocal.parseagentparameters(agentManual)
#        aflocal.parseagentparameters(agentSummary)
        aflocal.parseagentparameters(agentDisplay)
        aflocal.init()
        summary = aflocal.run(writeflags=True)
        aflocal.done()
        
        self.assertEqual(summary['report0']['spw:channel']['1:0']['flagged'], 1300)
        self.assertEqual(summary['report0']['spw:channel']['1:5']['flagged'], 1300)
        self.assertEqual(summary['report0']['spw:channel']['1:10']['flagged'], 1300)
        self.assertEqual(summary['report0']['spw:channel']['1:11']['flagged'], 0)
        self.assertEqual(summary['report0']['spw:channel']['1:60']['flagged'], 1300)
        self.assertEqual(summary['report0']['spw:channel']['1:59']['flagged'], 0)
        self.assertEqual(summary['report0']['flagged'], 15*1300)
Exemplo n.º 16
0
    def test_antint_simple_for_bpass_CalTable(self):
        '''Test the antenna integrations flagging mode on a bpass-based CalTable
        using MSSelection for antenna'''

        aflocal = aftool()
        aflocal.open(self.vis)
        aflocal.selectdata()
        
        agentUnflag = {'apply':True, 'mode':'unflag'}
        agentAntInt = {'apply': True, 'mode': 'antint', 'datacolumn': 'CPARAM',
                       'antint_ref_antenna': 'ea01', 'spw': '0', 'field': '0',
                       'minchanfrac': 0.7, 'verbose': True}
        agentSummary={'apply':True,'mode':'summary'}

        aflocal.parseagentparameters(agentUnflag)
        aflocal.parseagentparameters(agentAntInt)
        aflocal.parsesummaryparameters()
        aflocal.parseagentparameters(agentSummary)

        aflocal.init()
        summary = aflocal.run(writeflags=True)
        aflocal.done()

        self.assertEqual(summary['report0']['flagged'], 0)
        self.assertEqual(summary['report0']['total'], 1248000)
        self.assertEqual(summary['report0']['correlation']['Sol1']['flagged'], 0)
        self.assertEqual(summary['report0']['correlation']['Sol1']['total'], 624000)
        self.assertEqual(summary['report0']['correlation']['Sol2']['flagged'], 0)
        self.assertEqual(summary['report0']['correlation']['Sol2']['total'], 624000)
Exemplo n.º 17
0
    def test_mixed_agents_tsys(self):
        '''AgentFlagger: supported and unsupported agents in a list'''
        aflocal = aftool()
        aflocal.open(self.vis)
        aflocal.selectdata()
        aflocal.parsemanualparameters(spw='1')
        aflocal.parseagentparameters({
            'mode': 'shadow',
            'spw': '3'
        })  #unsupported mode
        aflocal.parseelevationparameters()  # unsupported mode
        aflocal.parsemanualparameters(spw='5')
        aflocal.parsesummaryparameters(spw='1,3,5', fieldcnt=True)
        aflocal.init()
        res = aflocal.run(writeflags=True)
        aflocal.done()

        # Check the summary dictionary with field breakdown
        fields = res['report0'].keys()
        fields.remove('name')
        fields.remove('type')
        fflags1 = 0
        fflags3 = 0
        fflags5 = 0
        totalflags = 0
        for ff in fields:
            fflags1 += res['report0'][ff]['spw']['1']['flagged']
            fflags3 += res['report0'][ff]['spw']['3']['flagged']
            fflags5 += res['report0'][ff]['spw']['5']['flagged']
            totalflags += res['report0'][ff]['flagged']

        self.assertEqual(fflags1, 32256)
        self.assertEqual(fflags3, 0)
        self.assertEqual(fflags5, 32256)
        self.assertEqual(totalflags, 32256 * 2)
Exemplo n.º 18
0
 def test_tfcrop_cparam_sol1_extension(self):
     """AgentFlagger:: Test tfcrop first calibration solution product of CPARAM column, 
     and then extend to the other solution for bpass CalTable"""
     aflocal = aftool()
     datacolumn = "CPARAM"
     correlation = 'Sol1'
     aflocal.open(self.vis)
     aflocal.selectdata()
     
     # Pre-clip data to avoid problems with near-zero values
     agentClip={'apply':True,'mode':'clip','clipzeros':True,'datacolumn':datacolumn,'correlation':correlation}
     aflocal.parseagentparameters(agentClip)
     
     aflocal.parsetfcropparameters(datacolumn=datacolumn, correlation=correlation,
                                   extendflags=False)
     aflocal.parsesummaryparameters()
     # Extend to the other solution
     aflocal.parseextendparameters(extendpols=True,growfreq=0.0,growtime=0.0)
     aflocal.parsesummaryparameters()
     aflocal.init()
     summary = aflocal.run(writeflags=True)
     aflocal.done()
     
     # flags from first summary, only tfcrop     
     assert abs(summary['report0']['flagged'] - 30427) <= 5        
     assert abs(summary['report0']['correlation']['Sol1']['flagged'] - 30427) <= 5
     
     # flags from second summary, tfcrop+extend
     assert abs(summary['report2']['flagged'] - 2*30427) <= 10
     assert abs(summary['report2']['correlation']['Sol2']['flagged'] - 30427) <= 5        
Exemplo n.º 19
0
    def test_mixed_agents_tsys(self):
        '''AgentFlagger: supported and unsupported agents in a list'''
        aflocal = aftool()
        aflocal.open(self.vis)
        aflocal.selectdata()
        aflocal.parsemanualparameters(spw='1')
        aflocal.parseagentparameters({'mode':'shadow','spw':'3'}) #unsupported mode
        aflocal.parseelevationparameters()     # unsupported mode
        aflocal.parsemanualparameters(spw='5')
        aflocal.parsesummaryparameters(spw='1,3,5', fieldcnt=True)
        aflocal.init()
        res = aflocal.run(writeflags=True)
        aflocal.done() 
        
        # Check the summary dictionary with field breakdown
        fields = res['report0'].keys()
        fields.remove('name')
        fields.remove('type')
        fflags1 = 0
        fflags3 = 0
        fflags5 = 0
        totalflags = 0
        for ff in fields:
          fflags1 += res['report0'][ff]['spw']['1']['flagged']
          fflags3 += res['report0'][ff]['spw']['3']['flagged']
          fflags5 += res['report0'][ff]['spw']['5']['flagged']
          totalflags += res['report0'][ff]['flagged']

        self.assertEqual(fflags1, 32256)
        self.assertEqual(fflags3, 0)
        self.assertEqual(fflags5, 32256)
        self.assertEqual(totalflags, 32256*2)
Exemplo n.º 20
0
 def test_unsupported_elevation_tsys(self):
     '''AgentFlagger: Unsupported elevation mode'''
     aflocal = aftool()
     aflocal.open(self.vis)
     aflocal.selectdata()
     aflocal.parseelevationparameters()
     aflocal.init()
     res = aflocal.run(writeflags=True)
     aflocal.done() 
     self.assertEqual(res, {})
Exemplo n.º 21
0
    def unflag_table(self):

        aflocal = aftool()
        aflocal.open(self.vis)
        aflocal.selectdata()
        agentUnflag = {'apply': True, 'mode': 'unflag'}
        aflocal.parseagentparameters(agentUnflag)
        aflocal.init()
        aflocal.run(writeflags=True)
        aflocal.done()
Exemplo n.º 22
0
 def test_unsupported_elevation_tsys(self):
     '''AgentFlagger: Unsupported elevation mode'''
     aflocal = aftool()
     aflocal.open(self.vis)
     aflocal.selectdata()
     aflocal.parseelevationparameters()
     aflocal.init()
     res = aflocal.run(writeflags=True)
     aflocal.done()
     self.assertEqual(res, {})
Exemplo n.º 23
0
    def unflag_table(self):

        aflocal = aftool()
        aflocal.open(self.vis)
        aflocal.selectdata()
        agentUnflag={'apply':True,'mode':'unflag'}
        aflocal.parseagentparameters(agentUnflag)
        aflocal.init()
        aflocal.run(writeflags=True)
        aflocal.done()
Exemplo n.º 24
0
 def test_default_cparam_bpass(self):
     '''Flagdata: flag CPARAM data column'''
     aflocal = aftool()
     aflocal.open(self.vis)
     aflocal.selectdata()
     aflocal.parseclipparameters(clipzeros=True,datacolumn='CPARAM')
     aflocal.parsesummaryparameters()
     aflocal.init()
     summary = aflocal.run(writeflags=True)
     aflocal.done() 
     self.assertEqual(summary['report0']['flagged'], 11078.0, 'Should use CPARAM as the default column')
Exemplo n.º 25
0
 def test_default_cparam_bpass(self):
     '''Flagdata: flag CPARAM data column'''
     aflocal = aftool()
     aflocal.open(self.vis)
     aflocal.selectdata()
     aflocal.parseclipparameters(clipzeros=True,datacolumn='CPARAM')
     aflocal.parsesummaryparameters()
     aflocal.init()
     summary = aflocal.run(writeflags=True)
     aflocal.done() 
     self.assertEqual(summary['report0']['flagged'], 11078.0, 'Should use CPARAM as the default column')
Exemplo n.º 26
0
 def test_display_float_data(self):
     '''AgentFlagger: Select spw, display and flag single-dish MS'''
     self.setUp_float_data()
     aflocal = aftool()
     aflocal.open(self.vis)
     aflocal.selectdata(spw='1~4')
     aflocal.parsemanualparameters()
     aflocal.parsesummaryparameters(name='Single-dish')
     agentDisplay={'mode':'display','datadisplay':True,'pause':False,'datacolumn':'FLOAT_DATA'}
     aflocal.parseagentparameters(agentDisplay)
     aflocal.init()
     summary = aflocal.run(writeflags=True)
     aflocal.done()
     self.assertEqual(summary['report0']['flagged'], 2052)
Exemplo n.º 27
0
    def test_flagversions(self):
        """Partition: check that the .flagversions is created"""

        # Run partition and create the .flagversions
        partition(vis=self.msfile, outputvis=self.mmsfile, createmms=True, disableparallel=True, datacolumn="data")
        self.assertTrue(os.path.exists(self.mmsfile + ".flagversions"))

        # Check that the number of backups in MMS is correct
        aflocal = aftool()
        aflocal.open(self.mmsfile)
        nv = aflocal.getflagversionlist()
        aflocal.done()
        self.assertEqual(len(nv), 3)

        # Run flagdata on MMS to check if it works well.
        flagdata(vis=self.mmsfile, mode="unflag", flagbackup=True)

        # Check that the number of backups in MMS is correct
        aflocal = aftool()
        aflocal.open(self.mmsfile)
        nvref = aflocal.getflagversionlist()
        aflocal.done()
        self.assertEqual(len(nvref), 4)
Exemplo n.º 28
0
 def test_display_float_data(self):
     '''AgentFlagger: Select spw, display and flag single-dish MS'''
     self.setUp_float_data()
     aflocal = aftool()
     aflocal.open(self.vis)
     aflocal.selectdata(spw='1~4')
     aflocal.parsemanualparameters()
     aflocal.parsesummaryparameters(name='Single-dish')
     agentDisplay={'mode':'display','datadisplay':True,'pause':False,'datacolumn':'FLOAT_DATA'}
     aflocal.parseagentparameters(agentDisplay)
     aflocal.init()
     summary = aflocal.run(writeflags=True)
     aflocal.done()
     self.assertEqual(summary['report0']['flagged'], 2052)
Exemplo n.º 29
0
 def test_clip_minmax_cparam_for_bpass(self):
     """AgentFlagger:: Clip all calibration solutions of CPARAM column using a minmax range"""
     aflocal = aftool()
     datacolumn = 'CPARAM'
     aflocal.open(self.vis)
     aflocal.selectdata()
     aflocal.parseclipparameters(clipminmax=[0.,0.3],datacolumn=datacolumn,clipzeros=True)
     aflocal.parsesummaryparameters()
     aflocal.init()
     summary = aflocal.run(writeflags=True)
     aflocal.done() 
     self.assertEqual(summary['report0']['flagged'], 11175.0)
     self.assertEqual(summary['report0']['total'], 1248000)
     self.assertEqual(summary['report0']['correlation']['Sol1']['flagged'], 11136.0)
     self.assertEqual(summary['report0']['correlation']['Sol2']['flagged'], 39)
Exemplo n.º 30
0
 def test_clip_minmax_cparam_for_bpass(self):
     """AgentFlagger:: Clip all calibration solutions of CPARAM column using a minmax range"""
     aflocal = aftool()
     datacolumn = 'CPARAM'
     aflocal.open(self.vis)
     aflocal.selectdata()
     aflocal.parseclipparameters(clipminmax=[0.,0.3],datacolumn=datacolumn,clipzeros=True)
     aflocal.parsesummaryparameters()
     aflocal.init()
     summary = aflocal.run(writeflags=True)
     aflocal.done() 
     self.assertEqual(summary['report0']['flagged'], 11175.0)
     self.assertEqual(summary['report0']['total'], 1248000)
     self.assertEqual(summary['report0']['correlation']['Sol1']['flagged'], 11136.0)
     self.assertEqual(summary['report0']['correlation']['Sol2']['flagged'], 39)
Exemplo n.º 31
0
    def test_manual_antenna_selection_agent_layer_for_tsys_CalTable2(self):
        """AgentFlagger:: Manually flag a Tsys-based CalTable using flag agent selection engine for antenna"""

        # Run the previous test but using the specific parsing functions
        aflocal = aftool()
        aflocal.open(self.vis)
        aflocal.selectdata()
        aflocal.parsemanualparameters(apply=False)
        aflocal.parsemanualparameters(antenna='DV09')
        aflocal.parsesummaryparameters()
        aflocal.init()
        summary = aflocal.run(writeflags=True)
        aflocal.done() 

        self.assertEqual(summary['report0']['antenna']['DV09']['flagged'], 14336)
        self.assertEqual(summary['report0']['antenna']['DV10']['flagged'], 0)
Exemplo n.º 32
0
 def test_display_data_single_channel_selection(self):
     """AgentFlagger:: Check nominal behaviour for single spw:chan selection """
     self.setUp_4Ants()
     aflocal = aftool()
     aflocal.open(self.vis)
     aflocal.selectdata()
     agentUnflag={'apply':True,'mode':'unflag'}
     agentManual={'apply':True,'mode':'manual','spw':'*:20~40'}
     agentSummary={'apply':True,'mode':'summary'}
     agentDisplay={'mode':'display','datadisplay':True,'pause':False}
     aflocal.parseagentparameters(agentManual)
     aflocal.parseagentparameters(agentSummary)
     aflocal.parseagentparameters(agentDisplay)
     aflocal.init()
     summary = aflocal.run(writeflags=True)
     aflocal.done()
Exemplo n.º 33
0
    def test_manual_antenna_selection_agent_layer_for_tsys_CalTable2(self):
        """AgentFlagger:: Manually flag a Tsys-based CalTable using flag agent selection engine for antenna"""

        # Run the previous test but using the specific parsing functions
        aflocal = aftool()
        aflocal.open(self.vis)
        aflocal.selectdata()
        aflocal.parsemanualparameters(apply=False)
        aflocal.parsemanualparameters(antenna='DV09')
        aflocal.parsesummaryparameters()
        aflocal.init()
        summary = aflocal.run(writeflags=True)
        aflocal.done() 

        self.assertEqual(summary['report0']['antenna']['DV09']['flagged'], 14336)
        self.assertEqual(summary['report0']['antenna']['DV10']['flagged'], 0)
Exemplo n.º 34
0
 def test_display_data_single_channel_selection(self):
     """AgentFlagger:: Check nominal behaviour for single spw:chan selection """
     self.setUp_4Ants()
     aflocal = aftool()
     aflocal.open(self.vis)
     aflocal.selectdata()
     agentUnflag = {'apply': True, 'mode': 'unflag'}
     agentManual = {'apply': True, 'mode': 'manual', 'spw': '*:20~40'}
     agentSummary = {'apply': True, 'mode': 'summary'}
     agentDisplay = {'mode': 'display', 'datadisplay': True, 'pause': False}
     aflocal.parseagentparameters(agentManual)
     aflocal.parseagentparameters(agentSummary)
     aflocal.parseagentparameters(agentDisplay)
     aflocal.init()
     summary = aflocal.run(writeflags=True)
     aflocal.done()
Exemplo n.º 35
0
 def test_null_intent_selection1(self):
     '''Agentflagger: handle unknown scan intent in list mode'''
     
     aflocal = aftool()
     aflocal.open(self.vis)
     aflocal.selectdata()
     aflocal.parsemanualparameters(intent='FOCUS') # non-existing intent
     aflocal.parsemanualparameters(intent='CALIBRATE_POINTING*') # scan=1
     aflocal.parsemanualparameters(intent='CALIBRATE_AMPLI_ON_SOURC') # typo
     aflocal.parsemanualparameters(intent='*DELAY*') # non-existing
     aflocal.parsesummaryparameters()
     aflocal.init()
     summary = aflocal.run()
     aflocal.done()
     self.assertEqual(summary['report0']['scan']['1']['flagged'], 192416)
     self.assertEqual(summary['report0']['scan']['8']['flagged'], 0)
     self.assertEqual(summary['report0']['flagged'], 192416)
Exemplo n.º 36
0
 def test_display_data_different_corrs_per_spw(self):
     """AgentFlagger:: Check behaviour when the number of correlation products changes between SPWs """
     self.setUp_CAS_4052()
     aflocal = aftool()
     aflocal.open(self.vis)
     aflocal.selectdata()
     agentUnflag={'apply':True,'mode':'unflag'}
     agentManual={'apply':True,'mode':'manual','spw':'*:100~200;300~400'}
     agentSummary={'apply':True,'mode':'summary'}
     agentDisplay={'mode':'display','datadisplay':True,'pause':False}
     aflocal.parseagentparameters(agentUnflag)
     aflocal.parseagentparameters(agentManual)
     aflocal.parseagentparameters(agentSummary)
     aflocal.parseagentparameters(agentDisplay)
     aflocal.init()
     summary = aflocal.run(writeflags=True)
     aflocal.done()
Exemplo n.º 37
0
 def test_null_intent_selection1(self):
     '''Agentflagger: handle unknown scan intent in list mode'''
     
     aflocal = aftool()
     aflocal.open(self.vis)
     aflocal.selectdata()
     aflocal.parsemanualparameters(intent='FOCUS') # non-existing intent
     aflocal.parsemanualparameters(intent='CALIBRATE_POINTING*') # scan=1
     aflocal.parsemanualparameters(intent='CALIBRATE_AMPLI_ON_SOURC') # typo
     aflocal.parsemanualparameters(intent='*DELAY*') # non-existing
     aflocal.parsesummaryparameters()
     aflocal.init()
     summary = aflocal.run()
     aflocal.done()
     self.assertEqual(summary['report0']['scan']['1']['flagged'], 192416)
     self.assertEqual(summary['report0']['scan']['8']['flagged'], 0)
     self.assertEqual(summary['report0']['flagged'], 192416)
Exemplo n.º 38
0
 def test_rflag_cparam_sol2(self):
     """AgentFlagger:: Test rflag solution 2 of CPARAM column for bpass"""
     aflocal = aftool()
     correlation = 'Sol2'
     aflocal.open(self.vis)
     aflocal.selectdata()
     agentRflag={'apply':True,'mode':'rflag','correlation':correlation,
                 'extendflags':False,'datacolumn':'CPARAM'}
     agentSummary={'apply':True,'mode':'summary'}
     aflocal.parseagentparameters(agentRflag)
     aflocal.parseagentparameters(agentSummary)
     aflocal.init()
     summary = aflocal.run(writeflags=True)
     aflocal.done() 
     self.assertEqual(summary['report0']['flagged'], 13197)
     self.assertEqual(summary['report0']['correlation']['Sol1']['flagged'], 0)
     self.assertEqual(summary['report0']['correlation']['Sol2']['flagged'], 13197)
Exemplo n.º 39
0
 def test_display_data_different_corrs_per_spw(self):
     """AgentFlagger:: Check behaviour when the number of correlation products changes between SPWs """
     self.setUp_CAS_4052()
     aflocal = aftool()
     aflocal.open(self.vis)
     aflocal.selectdata()
     agentUnflag={'apply':True,'mode':'unflag'}
     agentManual={'apply':True,'mode':'manual','spw':'*:100~200;300~400'}
     agentSummary={'apply':True,'mode':'summary'}
     agentDisplay={'mode':'display','datadisplay':True,'pause':False}
     aflocal.parseagentparameters(agentUnflag)
     aflocal.parseagentparameters(agentManual)
     aflocal.parseagentparameters(agentSummary)
     aflocal.parseagentparameters(agentDisplay)
     aflocal.init()
     summary = aflocal.run(writeflags=True)
     aflocal.done()
Exemplo n.º 40
0
 def test_rflag_cparam_sol2(self):
     """AgentFlagger:: Test rflag solution 2 of CPARAM column for bpass"""
     aflocal = aftool()
     correlation = 'Sol2'
     aflocal.open(self.vis)
     aflocal.selectdata()
     agentRflag={'apply':True,'mode':'rflag','correlation':correlation,
                 'extendflags':False,'datacolumn':'CPARAM'}
     agentSummary={'apply':True,'mode':'summary'}
     aflocal.parseagentparameters(agentRflag)
     aflocal.parseagentparameters(agentSummary)
     aflocal.init()
     summary = aflocal.run(writeflags=True)
     aflocal.done() 
     self.assertEqual(summary['report0']['flagged'], 13197)
     self.assertEqual(summary['report0']['correlation']['Sol1']['flagged'], 0)
     self.assertEqual(summary['report0']['correlation']['Sol2']['flagged'], 13197)
Exemplo n.º 41
0
    def test_manual_antenna_selection_agent_layer_for_tsys_CalTable(self):
        """AgentFlagger:: Manually flag a Tsys-based CalTable using flag agent selection engine for antenna"""
        aflocal = aftool()
        aflocal.open(self.vis)
        aflocal.selectdata()
        agentUnflag={'apply':True,'mode':'unflag'}
        agentManual={'apply':True,'mode':'manual','antenna':'DV09'}
        agentSummary={'apply':True,'mode':'summary'}
        aflocal.parseagentparameters(agentUnflag)
        aflocal.parseagentparameters(agentManual)
        aflocal.parseagentparameters(agentSummary)
        aflocal.init()
        summary = aflocal.run(writeflags=True)
        aflocal.done() 

        self.assertEqual(summary['report0']['antenna']['DV09']['flagged'], 14336)
        self.assertEqual(summary['report0']['antenna']['DV10']['flagged'], 0)
Exemplo n.º 42
0
    def test_manual_antenna_selection_agent_layer_for_bpass_CalTable(self):
        """AgentFlagger:: Manually flag a bpass-based CalTable using flag agent selection engine for antenna"""
        aflocal = aftool()
        aflocal.open(self.vis)
        aflocal.selectdata()
        agentUnflag={'apply':True,'mode':'unflag'}
        agentManual={'apply':True,'mode':'manual','antenna':'ea09'}
        agentSummary={'apply':True,'mode':'summary'}
        aflocal.parseagentparameters(agentUnflag)
        aflocal.parseagentparameters(agentManual)
        aflocal.parseagentparameters(agentSummary)
        aflocal.init()
        summary = aflocal.run(writeflags=True)
        aflocal.done() 

        self.assertEqual(summary['report0']['antenna']['ea09']['flagged'], 48000.0)
        self.assertEqual(summary['report0']['antenna']['ea10']['flagged'], 0.0)
Exemplo n.º 43
0
    def test_clip_minmax_fparm_sol1_extension(self):
        """AgentFlagger:: Test cliping first calibration solution product of FPARAM column using a minmax range, and then extend to the other solution """
        aflocal = aftool()
        datacolumn = 'FPARAM'
        aflocal.open(self.vis)
        aflocal.selectdata()
        agentUnflag={'apply':True,'mode':'unflag'}
        agentClip={'apply':True,'mode':'clip','clipzeros':True,'clipminmax':[0.,600.],'datacolumn':datacolumn,'correlation':'Sol1'}
        agentExtension={'apply':True,'mode':'extend','extendpols':True,'growfreq':0.0, 'growtime':0.0}
        agentSummary={'apply':True,'mode':'summary','correlation':'""'}
        aflocal.parseagentparameters(agentUnflag)
        aflocal.parseagentparameters(agentClip)
        aflocal.parseagentparameters(agentExtension)
        aflocal.parseagentparameters(agentSummary)
        aflocal.init()
        summary = aflocal.run(writeflags=True)
        aflocal.done() 

        self.assertEqual(summary['report0']['total'], 129024)
        self.assertEqual(summary['report0']['flagged'], 1500)
        self.assertEqual(summary['report0']['correlation']['Sol1']['flagged'], 750)
        self.assertEqual(summary['report0']['correlation']['Sol2']['flagged'], 750)
Exemplo n.º 44
0
    def test_clip_minmax_fparm_sol1_2(self):
        """AgentFlagger:: Test cliping first calibration solution product of FPARAM column using a minmax range """

        # Run the previous test but using the specific parsing functions
        aflocal = aftool()
        datacolumn = 'FPARAM'
        correlation = 'Sol1'
        aflocal.open(self.vis)
        aflocal.selectdata()
        aflocal.parsemanualparameters(apply=False)
        aflocal.parseclipparameters(clipzeros=True,clipminmax=[0.,600.],datacolumn=datacolumn,
                                     correlation=correlation)
        aflocal.parsesummaryparameters()
        aflocal.init()
        summary = aflocal.run(writeflags=True)
        aflocal.done() 
        self.assertEqual(summary['report0']['total'], 129024)
        self.assertEqual(summary['report0']['flagged'], 750)
        self.assertEqual(summary['report0']['correlation']['Sol1']['flagged'], 750)
        self.assertEqual(summary['report0']['correlation']['Sol1']['total'], 64512)
        self.assertEqual(summary['report0']['correlation']['Sol2']['flagged'], 0)
        self.assertEqual(summary['report0']['correlation']['Sol2']['total'], 64512)
Exemplo n.º 45
0
    def test_clip_minmax_fparm_sol1_extension(self):
        """AgentFlagger:: Test cliping first calibration solution product of FPARAM column using a minmax range, and then extend to the other solution """
        aflocal = aftool()
        datacolumn = 'FPARAM'
        aflocal.open(self.vis)
        aflocal.selectdata()
        agentUnflag={'apply':True,'mode':'unflag'}
        agentClip={'apply':True,'mode':'clip','clipzeros':True,'clipminmax':[0.,600.],'datacolumn':datacolumn,'correlation':'Sol1'}
        agentExtension={'apply':True,'mode':'extend','extendpols':True,'growfreq':0.0, 'growtime':0.0}
        agentSummary={'apply':True,'mode':'summary','correlation':'""'}
        aflocal.parseagentparameters(agentUnflag)
        aflocal.parseagentparameters(agentClip)
        aflocal.parseagentparameters(agentExtension)
        aflocal.parseagentparameters(agentSummary)
        aflocal.init()
        summary = aflocal.run(writeflags=True)
        aflocal.done() 

        self.assertEqual(summary['report0']['total'], 129024)
        self.assertEqual(summary['report0']['flagged'], 1500)
        self.assertEqual(summary['report0']['correlation']['Sol1']['flagged'], 750)
        self.assertEqual(summary['report0']['correlation']['Sol2']['flagged'], 750)
Exemplo n.º 46
0
 def test_summarylist1(self):
     '''agentflagger: multiple summaries'''
     aflocal = aftool()
     aflocal.open(self.vis)
     aflocal.selectdata()
     aflocal.parsesummaryparameters(name='summary_1')
     aflocal.parsemanualparameters(intent='FOCUS') # non-existing intent
     aflocal.parsesummaryparameters(name='summary_2')
     aflocal.parsemanualparameters(spw='0') 
     aflocal.parsesummaryparameters(name='summary_3')
     aflocal.init()
     summary = aflocal.run()
     aflocal.done()
     
     # It creates 2 reports per summary agent. One is type='summary',
     # ther other is type='plotpoints'
     self.assertEqual(summary['report0']['name'],'summary_1')
     self.assertEqual(summary['report2']['name'],'summary_2')
     self.assertEqual(summary['report4']['name'],'summary_3')
     self.assertEqual(summary['report0']['flagged'], 0)
     self.assertEqual(summary['report2']['flagged'], 0)
     self.assertEqual(summary['report4']['spw']['0']['flagged'], summary['report4']['spw']['0']['total'])
Exemplo n.º 47
0
    def test_clip_minmax_fparm_all(self):
        """AgentFlagger:: Test cliping all calibration solution products of FPARAM column using a minmax range """
        aflocal = aftool()
        datacolumn = 'FPARAM'
        correlation = ''
        aflocal.open(self.vis)
        aflocal.selectdata()
        agentUnflag={'apply':True,'mode':'unflag'}
        agentClip={'apply':True,'mode':'clip','clipzeros':True,'clipminmax':[0.,600.],'datacolumn':datacolumn,'correlation':correlation}
        agentSummary={'apply':True,'mode':'summary'}
        aflocal.parseagentparameters(agentUnflag)
        aflocal.parseagentparameters(agentClip)
        aflocal.parseagentparameters(agentSummary)
        aflocal.init()
        summary = aflocal.run(writeflags=True)
        aflocal.done() 

        self.assertEqual(summary['report0']['total'], 129024.0)
        self.assertEqual(summary['report0']['flagged'], 1192.0)
        self.assertEqual(summary['report0']['correlation']['Sol1']['flagged'], 750.0)
        self.assertEqual(summary['report0']['correlation']['Sol1']['total'], 64512.0)
        self.assertEqual(summary['report0']['correlation']['Sol2']['flagged'], 442.0)
        self.assertEqual(summary['report0']['correlation']['Sol2']['total'], 64512.0)
Exemplo n.º 48
0
    def test_clip_nan_and_inf_fparm_all(self):
        """AgentFlagger:: Test cliping only NaNs/Infs in all calibration solution products of FPARAM column"""
        aflocal = aftool()
        datacolumn = 'FPARAM'
        correlation = ''
        aflocal.open(self.vis)
        aflocal.selectdata()
        agentUnflag={'apply':True,'mode':'unflag'}
        agentClip={'apply':True,'mode':'clip','datacolumn':datacolumn,'correlation':correlation}
        agentSummary={'apply':True,'mode':'summary'}
        aflocal.parseagentparameters(agentUnflag)
        aflocal.parseagentparameters(agentClip)
        aflocal.parseagentparameters(agentSummary)
        aflocal.init()
        summary = aflocal.run(writeflags=True)
        aflocal.done() 

        self.assertEqual(summary['report0']['total'], 129024.0)
        self.assertEqual(summary['report0']['flagged'], 0.0)
        self.assertEqual(summary['report0']['correlation']['Sol1']['flagged'], 0.0)
        self.assertEqual(summary['report0']['correlation']['Sol1']['total'], 64512.0)
        self.assertEqual(summary['report0']['correlation']['Sol2']['flagged'], 0.0)
        self.assertEqual(summary['report0']['correlation']['Sol2']['total'], 64512.0)
Exemplo n.º 49
0
    def test_clip_minmax_fparm_error_case_absall(self):
        """AgentFlagger:: Error case test when a complex operator is used with CalTables """
        aflocal = aftool()
        datacolumn = 'FPARAM'
        correlation = 'ABS_ALL'
        aflocal.open(self.vis)
        aflocal.selectdata()
        agentUnflag={'apply':True,'mode':'unflag'}
        agentClip={'apply':True,'mode':'clip','clipzeros':True,'clipminmax':[0.,600.],'datacolumn':datacolumn,'correlation':correlation}
        agentSummary={'apply':True,'mode':'summary'}
        aflocal.parseagentparameters(agentUnflag)
        aflocal.parseagentparameters(agentClip)
        aflocal.parseagentparameters(agentSummary)
        aflocal.init()
        summary = aflocal.run(writeflags=True)
        aflocal.done() 

        self.assertEqual(summary['report0']['total'], 129024.0)
        self.assertEqual(summary['report0']['flagged'], 1192.0)
        self.assertEqual(summary['report0']['correlation']['Sol1']['flagged'], 750.0)
        self.assertEqual(summary['report0']['correlation']['Sol1']['total'], 64512.0)
        self.assertEqual(summary['report0']['correlation']['Sol2']['flagged'], 442.0)
        self.assertEqual(summary['report0']['correlation']['Sol2']['total'], 64512.0)
Exemplo n.º 50
0
    def test_clip_minmax_snr_all_for_bpass_CalTable(self):
        """AgentFlagger:: Test cliping all calibration solution products of SNR column using a minmax range for bpass CalTable"""
        aflocal = aftool()
        datacolumn = 'SNR'
        correlation = ''
        aflocal.open(self.vis)
        aflocal.selectdata()
        agentUnflag={'apply':True,'mode':'unflag'}
        agentClip={'apply':True,'mode':'clip','clipzeros':True,'clipminmax':[0.,550.],'datacolumn':datacolumn,'correlation':correlation}
        agentSummary={'apply':True,'mode':'summary'}
        aflocal.parseagentparameters(agentUnflag)
        aflocal.parseagentparameters(agentClip)
        aflocal.parseagentparameters(agentSummary)
        aflocal.init()
        summary = aflocal.run(writeflags=True)
        aflocal.done() 

        self.assertEqual(summary['report0']['total'], 1248000.0)
        self.assertEqual(summary['report0']['flagged'], 74371.0)
        self.assertEqual(summary['report0']['correlation']['Sol1']['flagged'], 36327.0)
        self.assertEqual(summary['report0']['correlation']['Sol1']['total'], 624000.0)
        self.assertEqual(summary['report0']['correlation']['Sol2']['flagged'], 38044.0)
        self.assertEqual(summary['report0']['correlation']['Sol2']['total'], 624000.0)
Exemplo n.º 51
0
    def test_manual_antenna_msSelection_layer_for_tsys_CalTable(self):
        """AgentFlagger:: Manually flag a Tsys-based CalTable using MSSelection for antenna"""
        aflocal = aftool()

        aflocal.open(self.vis)
        aflocal.selectdata(antenna='DV09')
        agentUnflag={'apply':True,'mode':'unflag'}
        agentManual={'apply':True,'mode':'manual'}
        aflocal.parseagentparameters(agentUnflag)
        aflocal.parseagentparameters(agentManual)
        aflocal.init()
        summary = aflocal.run(writeflags=True)
        aflocal.done() 

        aflocal.open(self.vis)
        aflocal.selectdata()
        agentSummary={'apply':True,'mode':'summary'}
        aflocal.parseagentparameters(agentSummary)
        aflocal.init()
        summary = aflocal.run(writeflags=True)
        aflocal.done() 

        self.assertEqual(summary['report0']['antenna']['DV09']['flagged'], 14336)
        self.assertEqual(summary['report0']['antenna']['DV10']['flagged'], 0)
Exemplo n.º 52
0
        os.system('rm -f '+ inp)
        
    # save to a file    
    with open(inp, 'w') as f:
        f.write(cmd)
        
    f.close()
    
    return

    
# Path for data
datapath = os.environ.get('CASAPATH').split()[0] + "/data/regression/unittest/flagdata/"

# Local copy of the agentflagger tool
aflocal = aftool()

# Base class which defines setUp functions
# for importing different data sets
class test_base(unittest.TestCase):
    
    def setUp_ngc5921(self):
        self.vis = "ngc5921.ms"

        if os.path.exists(self.vis):
            print "The MS is already around, just unflag"
        else:
            os.system('cp -r '+datapath + self.vis +' '+ self.vis)

        os.system('rm -rf ' + self.vis + '.flagversions')
        self.unflag_ms()        
Exemplo n.º 53
0
# Pick up alternative data directory to run tests on MMSs
testmms = False
if os.environ.has_key('TEST_DATADIR'):   
    DATADIR = str(os.environ.get('TEST_DATADIR'))+'/flagdata/'
    if os.path.isdir(DATADIR):
        testmms = True
        datapath = DATADIR

print 'flagmanager tests will use data from '+datapath         

# jagonzal (CAS-4287): Add a cluster-less mode to by-pass parallel processing for MMSs as requested 
if os.environ.has_key('BYPASS_PARALLEL_PROCESSING'):
    ParallelTaskHelper.bypassParallelProcessing(1)

# Local copy of the agentflagger tool
aflocal = aftool()

# Base class which defines setUp functions for importing different data sets
class test_base(unittest.TestCase):
    
    def setUp_flagdatatest(self):
        '''VLA data set, scan=2500~2600 spw=0 1 chan, RR,LL'''
        self.vis = "flagdatatest.ms"
        if testmms:
            self.vis = "flagdatatest.mms"

        if os.path.exists(self.vis):
            print "The MS is already around, just unflag"
        else:
            print "Moving data..."
            os.system('cp -r '+datapath + self.vis +' '+ self.vis)
Exemplo n.º 54
0
def flagdata(
        vis,
        mode,
        autocorr,  # mode manual parameter
        inpfile,  # mode list parameters
        reason,
        tbuff,
        spw,  # data selection parameters
        field,
        antenna,
        uvrange,
        timerange,
        correlation,
        scan,
        intent,
        array,
        observation,
        feed,
        clipminmax,  # mode clip parameters
        datacolumn,
        clipoutside,
        channelavg,
        chanbin,
        timeavg,
        timebin,
        clipzeros,
        quackinterval,  # mode quack parameters
        quackmode,
        quackincrement,
        tolerance,  # mode shadow parameter
        addantenna,
        lowerlimit,  # mode elevation parameters
        upperlimit,
        ntime,  # mode tfcrop
        combinescans,
        timecutoff,
        freqcutoff,
        timefit,
        freqfit,
        maxnpieces,
        flagdimension,
        usewindowstats,
        halfwin,
        extendflags,
        winsize,  # rflag parameters
        timedev,
        freqdev,
        timedevscale,
        freqdevscale,
        spectralmax,
        spectralmin,
        antint_ref_antenna,  # mode antint
        minchanfrac,
        verbose,
        extendpols,  # mode extend
        growtime,
        growfreq,
        growaround,
        flagneartime,
        flagnearfreq,
        minrel,  # mode summary
        maxrel,
        minabs,
        maxabs,
        spwchan,
        spwcorr,
        basecnt,
        fieldcnt,
        name,
        action,  # run or not the tool
        display,
        flagbackup,
        savepars,  # save the current parameters to FLAG_CMD  or to a file
        cmdreason,  # reason to save to flag cmd
        outfile,  # output file to save flag commands
        overwrite,  # overwrite the outfile file
        writeflags  # HIDDEN parameter
):

    #
    # Task flagdata
    #    Flags data from an MS or calibration table based on data selection in various ways

    casalog.origin('flagdata')

    if (action == 'none' or action == '' or action == 'calculate'):
        flagbackup = False

    # SMC: moved the flagbackup to before initializing the cluster.
    # Note that with this change, a flag backup will be created even if
    # an error happens that prevents the flagger tool from running.
    if (mode != 'summary' and flagbackup):
        casalog.post('Backup original flags before applying new flags')
        fh.backupFlags(aflocal=None, msfile=vis, prename='flagdata')
        # Set flagbackup to False because only the controller
        # should create a backup
        flagbackup = False

    # Initialize the helper class
    orig_locals = copy.deepcopy(locals())
    FHelper = FlagHelper()

    # Check if vis is a MS, MMS or cal table:
    # typevis = 1 --> cal table
    # typevis = 0 --> MS
    # typevis = 2 --> MMS
    iscal = False
    typevis = fh.isCalTable(vis)
    if typevis == 1:
        iscal = True

    # ***************** Input is MMS -- Parallel Processing ***********************

    if FHelper.isMPIEnabled(
    ) and typevis == 2 and action != '' and action != 'none':

        # Create a temporary input file with .tmp extension.
        # Use this file for all the processing from now on.
        if (isinstance(inpfile,str) and inpfile != '') or \
           (isinstance(inpfile, list) and os.path.isfile(inpfile[0])):
            inpfile = FHelper.setupInputFile(inpfile)
            if inpfile != None:
                orig_locals['inpfile'] = inpfile

        if outfile != '':
            outfile = os.path.abspath(outfile)
            orig_locals['outfile'] = outfile
        if isinstance(addantenna, str) and addantenna != '':
            addantenna = os.path.abspath(addantenna)
            orig_locals['addantenna'] = addantenna
        if isinstance(timedev, str) and timedev != '':
            timedev = os.path.abspath(timedev)
            orig_locals['timedev'] = timedev
        if isinstance(freqdev, str) and freqdev != '':
            freqdev = os.path.abspath(freqdev)
            orig_locals['freqdev'] = freqdev

        FHelper.__init__(orig_locals)

        # For tests only
        # FHelper.bypassParallelProcessing(1)

        FHelper.setupCluster('flagdata')
        # (CAS-4119): Override summary minabs,maxabs,minrel,maxrel
        # so that it is done after consolidating the summaries

        # By-pass options to filter summary
        filterSummary = False
        if ((mode == 'summary') and ((minrel != 0.0) or (maxrel != 1.0) or
                                     (minabs != 0) or (maxabs != -1))):
            filterSummary = True

            myms = mstool()
            myms.open(vis)
            subMS_list = myms.getreferencedtables()
            myms.close()

            if (minrel != 0.0):
                minreal_dict = create_arg_dict(subMS_list, 0.0)
                FHelper.override_arg('minrel', minreal_dict)
            if (maxrel != 1.0):
                maxrel_dict = create_arg_dict(subMS_list, 1.0)
                FHelper.override_arg('maxrel', maxrel_dict)
            if (minabs != 0):
                minabs_dict = create_arg_dict(subMS_list, 0)
                FHelper.override_arg('minabs', minabs_dict)
            if (maxabs != -1):
                maxabs_dict = create_arg_dict(subMS_list, -1)
                FHelper.override_arg('maxabs', maxabs_dict)

        # By-pass options to filter summary
        if savepars:

            myms = mstool()
            myms.open(vis)
            subMS_list = myms.getreferencedtables()
            myms.close()

            savepars_dict = create_arg_dict(subMS_list, False)
            FHelper.override_arg('savepars', savepars_dict)

        # Execute the parallel engines
        retVar = FHelper.go()

        # Save overall RFlag values into timedev/freqdev files once the subMS results have
        # been consolidated, if needed for RFlag (introduced in CAS-11850)
        opts_dict = {
            'writeflags': writeflags,
            'timedev': timedev,
            'freqdev': freqdev
        }
        fh.save_rflag_consolidated_files(mode, action, retVar, opts_dict,
                                         inpfile)

        # In async mode return the job ids
        if ParallelTaskHelper.getAsyncMode():
            return retVar
        else:
            # Filter summary at MMS level
            if (mode == 'summary'):
                if filterSummary:
                    retVar = filter_summary(retVar, minrel, maxrel, minabs,
                                            maxabs)
                return retVar
            # Save parameters at MMS level
            elif savepars:
                action = 'none'
            else:
                return retVar

    summary_stats = {}

    # ***************** Input is a normal MS/cal table ****************

    # Create local tools
    #    aflocal = casac.agentflagger()
    aflocal = aftool()
    mslocal = mstool()

    try:
        # Verify the ntime value
        newtime = 0.0
        if type(ntime) == float or type(ntime) == int:
            if ntime <= 0:
                raise Exception, 'Parameter ntime cannot be < = 0'
            else:
                # units are seconds
                newtime = float(ntime)

        elif type(ntime) == str:
            if ntime == 'scan':
                # iteration time step is a scan
                newtime = 0.0
            else:
                # read the units from the string
                qtime = qa.quantity(ntime)

                if qtime['unit'] == 'min':
                    # convert to seconds
                    qtime = qa.convert(qtime, 's')
                elif qtime['unit'] == '':
                    qtime['unit'] = 's'

                # check units
                if qtime['unit'] == 's':
                    newtime = qtime['value']
                else:
                    casalog.post(
                        'Cannot convert units of ntime. Will use default 0.0s',
                        'WARN')

        casalog.post(
            "New ntime is of type %s and value %s" % (type(newtime), newtime),
            'DEBUG')

        # Open the MS and attach it to the tool
        if ((type(vis) == str) & (os.path.exists(vis))):
            aflocal.open(vis, newtime)
        else:
            raise Exception, 'Visibility data set not found - please verify the name'

        # Get the parameters for the mode
        agent_pars = {}

        # By default, write flags to the MS
        writeflags = True

        # Only the apply action writes to the MS
        # action=apply     --> write to the MS
        # action=calculate --> do not write to the MS
        # action=''        --> do not run the tool and do not write to the MS
        if action != 'apply':
            writeflags = False

        # Default mode
        if mode == '' or mode == 'manualflag':
            mode = 'manual'

        # Read in the list of commands
        # Make a dictionary of the input commands. Select by reason if requested
        flagcmd = {}

        if mode == 'list':
            casalog.post('List mode is active')
            doPadding = True
            try:
                # If tbuff is requested, read and Parse
                if tbuff == 0.0 or tbuff == [] or tbuff == None:
                    doPadding = False

                if doPadding:
                    casalog.post('Will apply time buffer padding')

                    # inpfile is a file
                    if isinstance(inpfile, str):
                        inpfile = [inpfile]

                    # read in the list and do a simple parsing to apply tbuff
                    flaglist = fh.readAndParse(inpfile, tbuff)

                else:
                    flaglist = fh.get_flag_cmd_list(inpfile)

                # Parse and create a dictionary
                flagcmd = fh.parseDictionary(flaglist, reason)

                # Validate the dictionary.
                # IMPORTANT: if any parameter changes its type, the following
                # function needs to be updated. The same if any new parameter is
                # added or removed from the task
                fh.evaluateFlagParameters(flagcmd, orig_locals)

                # List of flag commands in dictionary
                vrows = flagcmd.keys()

                casalog.post('%s' % flagcmd, 'DEBUG1')

            except Exception, instance:
                casalog.post('%s' % instance, 'ERROR')
                raise Exception, 'Error reading the input list. Make sure the syntax used in the list '\
                                 'follows the rules given in the inline help of the task.'

            casalog.post('Selected ' + str(vrows.__len__()) +
                         ' commands from combined input list(s) ')

        elif mode == 'manual':
            agent_pars['autocorr'] = autocorr
            casalog.post('Manual mode is active')