def test_split_MMS(self):
        '''mstransform: Split MMS in parallel'''
        # Create an MMS in the setup. It creates self.testmms
        self.createMMS(self.vis, axis='scan', spws='0,1')
        
        self.outputms = 'scan30.mms'
        mstransform(vis=self.testmms, outputvis=self.outputms, datacolumn='data', scan='30')
        
        self.assertTrue(ParallelTaskHelper.isParallelMS(self.outputms),'Output is not an MMS')
        
        mslocal = mstool()
        mslocal.open(self.outputms)
        sublist = mslocal.getreferencedtables()
        self.assertEqual(len(sublist), 1)
        
        # Test DD table
        msmdt = msmdtool()
        msmdt.open(self.outputms)
        out_dds = msmdt.datadescids()
        msmdt.done()
        
        ref = [0,1]
        for i in out_dds:
            self.assertEqual(out_dds[i], ref[i])

        # The separation axis should be copied to the output MMS
        in_sepaxis = ph.axisType(self.testmms)
        out_sepaxis = ph.axisType(self.outputms)
        self.assertEqual(in_sepaxis, out_sepaxis, 'AxisTypes from input and output MMS do not match')
    def test_MMS1(self):
        '''mstransform: input MMS should be the same as output MMS'''
        
        # Create an MMS in the setup
        self.createMMS(self.vis, axis='scan', spws='0,1')
                
        # Create another MS and compare. They should be the same
        self.outputms = 'thesame.mms'
        mstransform(vis=self.testmms, outputvis=self.outputms, datacolumn='data')
        
        self.assertTrue(ParallelDataHelper.isParallelMS(self.outputms),'Output is not an MMS')
                
        # Sort the MSs so that they can be compared
        myms = mstool()
        
        myms.open(self.testmms)
        myms.sort('input_sorted.ms',['OBSERVATION_ID','ARRAY_ID','SCAN_NUMBER','FIELD_ID','DATA_DESC_ID','ANTENNA1','ANTENNA2','TIME'])
        myms.done()
        
        myms.open(self.outputms)
        myms.sort('output_sorted.ms',['OBSERVATION_ID','ARRAY_ID','SCAN_NUMBER','FIELD_ID','DATA_DESC_ID','ANTENNA1','ANTENNA2','TIME'])
        myms.done()

        # Compare both tables. Ignore the DATA column and compare it in next line
        self.assertTrue(th.compTables('input_sorted.ms','output_sorted.ms', 
                                      ['FLAG_CATEGORY','FLAG','WEIGHT_SPECTRUM','SIGMA_SPECTRUM','DATA']))
        
        # Compare the DATA column
        self.assertTrue(th.compVarColTables('input_sorted.ms','output_sorted.ms','DATA'))
        
        # The separation axis should be copied to the output MMS
        in_sepaxis = ph.axisType(self.testmms)
        out_sepaxis = ph.axisType(self.outputms)
        self.assertEqual(in_sepaxis, out_sepaxis, 'AxisTypes from input and output MMS do not match')
    def test_split_MMS_weight_corr_sel(self):
        '''mstransform: Split MMS in parallel. Check WEIGHT shape when selecting correlation'''
        # Create an MMS in the setup. It creates self.testmms
        self.createMMS(self.vis, axis='scan', spws='0,1')
        
        self.outputms = 'corrRR_LL.mms'
        mstransform(vis=self.testmms, outputvis=self.outputms, datacolumn='data', correlation='RR,LL',spw='0')
        
        self.assertTrue(ParallelTaskHelper.isParallelMS(self.outputms),'Output is not an MMS')
        
        mslocal = mstool()
        mslocal.open(self.outputms)
        sublist = mslocal.getreferencedtables()
        self.assertEqual(len(sublist), 2)
        
        # Test DD table
        msmdt = msmdtool()
        msmdt.open(self.outputms)
        out_dds = msmdt.datadescids()
        msmdt.done()
        
        ref = [0]
        for i in out_dds:
            self.assertEqual(out_dds[i], ref[i])

        # The separation axis should be copied to the output MMS
        in_sepaxis = ph.axisType(self.testmms)
        out_sepaxis = ph.axisType(self.outputms)
        self.assertEqual(in_sepaxis, out_sepaxis, 'AxisTypes from input and output MMS do not match')

        # Check the dimensions of the WEIGHT and SIGMA columns. CAS-6946
        out_ws = th.getColShape(self.outputms,'WEIGHT')
        out_ss = th.getColShape(self.outputms,'SIGMA')
        self.assertEqual(out_ws[0],'[2]','WEIGHT shape is not correct')
        self.assertEqual(out_ss[0],'[2]','SIGMA shape is not correct')
Exemple #4
0
    def test_default_sequential(self):
        '''Partition: create an MMS with default values in sequential'''
        partition(vis=self.msfile, outputvis=self.mmsfile, disableparallel=True)
        
        self.assertTrue(os.path.exists(self.mmsfile), 'MMS was not created for this test')
        
        # Take the dictionary and compare with original MS
        thisdict = listpartition(vis=self.mmsfile, createdict=True)
        
        # Get scans of MMS
        slist = ph.getMMSScans(thisdict)
         
        for s in slist:
            mmsN = ph.getMMSScanNrows(thisdict, s)
            msN = ph.getScanNrows(self.msfile, s)
            self.assertEqual(mmsN, msN, 'Nrows in scan=%s differs: mms_nrows=%s <--> ms_nrows=%s'
                             %(s, mmsN, msN))
 
        # Compare spw IDs of MS and MMS
        for s in slist:
            mms_spw = ph.getSpwIds(self.mmsfile, s)
            ms_spw = ph.getSpwIds(self.msfile, s)
            self.assertEqual(mms_spw, ms_spw, 'list of spws in scan=%s differs: '\
                             'mms_spw=%s <--> ms_spw=%s' %(s, mmsN, msN))
            
        # The separation axis should be written to the output MMS
        sepaxis = ph.axisType(self.mmsfile)
        self.assertEqual(sepaxis, 'scan,spw', 'Partition did not write AxisType correctly in MMS')
Exemple #5
0
    def test_sepaxis(self):
        '''Partition: separationaxis=auto'''        
        partition(vis=self.msfile, outputvis=self.mmsfile, spw='0~11',separationaxis='auto',
                  flagbackup=False, disableparallel=True, datacolumn='data')
#        partition(vis=self.msfile, outputvis=self.mmsfile,separationaxis='auto')
        
        self.assertTrue(os.path.exists(self.mmsfile), 'MMS was not created for this test')

        # Dictionary with selection to compare with original MS
        mysel = {'spw':'0~11'}
        
        # Take the dictionary and compare with original MS
        thisdict = listpartition(vis=self.mmsfile, createdict=True)
        
        # Compare nrows of all scans in selection
        slist = ph.getMMSScans(thisdict)
        self.assertEqual(slist.__len__(), 2)
        for s in slist:
            mmsN = ph.getMMSScanNrows(thisdict, s)
            msN = ph.getScanNrows(self.msfile, s, selection=mysel)
#            msN = ph.getScanNrows(self.msfile, s)
            self.assertEqual(mmsN, msN, 'Nrows in scan=%s differs: mms_nrows=%s <--> ms_nrows=%s'
                             %(s, mmsN, msN))

        # Compare spw IDs
        for s in slist:
            mms_spw = ph.getSpwIds(self.mmsfile, s)
            ms_spw = ph.getSpwIds(self.msfile, s, selection=mysel)
#            ms_spw = ph.getSpwIds(self.msfile, s)
            self.assertEqual(mms_spw, ms_spw, 'list of spws in scan=%s differs: '\
                             'mms_spw=%s <--> ms_spw=%s' %(s, mms_spw, ms_spw))

        # The separation axis should be written to the output MMS
        sepaxis = ph.axisType(self.mmsfile)
        self.assertEqual(sepaxis, 'scan,spw', 'Partition did not write AxisType correctly in MMS')
Exemple #6
0
    def test_scan_spw(self):
        '''Partition: separationaxis=scan with spw selection'''
        partition(vis=self.msfile, outputvis=self.mmsfile, separationaxis='scan',
                  spw='1~4,10,11', flagbackup=False, datacolumn='data')
        
        self.assertTrue(os.path.exists(self.mmsfile), 'MMS was not created for this test')
        self.assertTrue(os.path.exists(self.msfile),'Make sure the input MS is not deleted inside the task')
        
        # Take the dictionary and compare with original MS
        thisdict = listpartition(vis=self.mmsfile, createdict=True)

        # Dictionary with selection to compare with original MS
        mysel = {'spw':'1~4,10,11'}
        
        # Compare nrows of all scans in selection
        slist = ph.getMMSScans(thisdict)
        for s in slist:
            mmsN = ph.getMMSScanNrows(thisdict, s)
            msN = ph.getScanNrows(self.msfile, s, selection=mysel)
            self.assertEqual(mmsN, msN, 'Nrows in scan=%s differs: mms_nrows=%s <--> ms_nrows=%s'
                             %(s, mmsN, msN))

        # The comparison should be
        # ms_spw = 1 --> mms_spw = 0
        # ms_spw = 2 --> mms_spw = 1, etc.
        # Check that MMS spw IDs have been re-indexed properly
        indexed_ids = range(6)
        for s in slist:
            mms_spw = ph.getSpwIds(self.mmsfile, s)
            self.assertEqual(mms_spw, indexed_ids, 'spw IDs were not properly re-indexed')
            
        # The separation axis should be written to the output MMS
        sepaxis = ph.axisType(self.mmsfile)
        self.assertEqual(sepaxis, 'scan', 'Partition did not write AxisType correctly in MMS')
Exemple #7
0
    def test_default_sequential(self):
        '''Partition: create an MMS with default values in sequential'''
        partition(vis=self.msfile,
                  outputvis=self.mmsfile,
                  disableparallel=True)

        self.assertTrue(os.path.exists(self.mmsfile),
                        'MMS was not created for this test')

        # Take the dictionary and compare with original MS
        thisdict = listpartition(vis=self.mmsfile, createdict=True)

        # Get scans of MMS
        slist = ph.getMMSScans(thisdict)

        for s in slist:
            mmsN = ph.getMMSScanNrows(thisdict, s)
            msN = ph.getScanNrows(self.msfile, s)
            self.assertEqual(
                mmsN, msN,
                'Nrows in scan=%s differs: mms_nrows=%s <--> ms_nrows=%s' %
                (s, mmsN, msN))

        # Compare spw IDs of MS and MMS
        for s in slist:
            mms_spw = ph.getSpwIds(self.mmsfile, s)
            ms_spw = ph.getSpwIds(self.msfile, s)
            self.assertEqual(mms_spw, ms_spw, 'list of spws in scan=%s differs: '\
                             'mms_spw=%s <--> ms_spw=%s' %(s, mmsN, msN))

        # The separation axis should be written to the output MMS
        sepaxis = ph.axisType(self.mmsfile)
        self.assertEqual(sepaxis, 'scan,spw',
                         'Partition did not write AxisType correctly in MMS')
Exemple #8
0
 def test_baseline1(self):
     '''partition: create an MMS per baseline axis. Use the number of MPI servers'''
     self.outputms = 'baseline1.mms'
     partition(self.vis, outputvis=self.outputms, separationaxis='baseline', flagbackup=False)
     
     # The separation axis should be written to the output MMS
     sepaxis = ph.axisType(self.outputms)
     self.assertEqual(sepaxis, 'baseline', 'Partition did not write AxisType correctly in MMS')
    def test_mms3(self):
        '''test_mms3: Create MMS with separationaxis=spw and lazy=True'''
        myasdmname = 'uid___A002_X71e4ae_X317_short'
        themsname = myasdmname+".ms"

        importasdm(myasdmname, createmms=True, lazy=True, scans='1,2', separationaxis='spw', flagbackup=False,
                   process_flags=False)
        self.assertTrue(ParallelDataHelper.isParallelMS(themsname), 'Output is not a Multi-MS')
        self.assertEqual(ph.axisType(themsname), 'spw', 'Separation axis of MMS should be spw')
Exemple #10
0
    def test_MMS_as_monolithicMS(self):
        '''mstransform: MMS should be processed as a monolithic MS'''
        # Create an MMS in the setup. It creates self.testmms
        self.createMMS(self.vis, axis='spw', spws='2,4,6')
        
        self.outputms = 'monolithicMMS.mms'
        # Treat MMS as a monolithic MS and create an output MMS with different separation axis.
        mstransform(vis=self.testmms, outputvis=self.outputms, datacolumn='data', combinespws=True)
        self.assertTrue(ParallelDataHelper.isParallelMS(self.outputms),'Output should be an MMS')
        
        # The separation axis should be copied to the output MMS
        in_sepaxis = ph.axisType(self.testmms)
        out_sepaxis = ph.axisType(self.outputms)
        self.assertNotEqual(in_sepaxis, out_sepaxis, 'AxisTypes from input and output MMS should not match')        

        ret = th.verifyMS(self.outputms, 1, 320, 0)
        self.assertTrue(ret[0],ret[1])

        listobs(self.outputms, listfile='list1.obs')
        self.assertTrue(os.path.exists('list1.obs'), 'Probable error in sub-table re-indexing')
Exemple #11
0
    def test_baseline1(self):
        '''partition: create an MMS per baseline axis. Use the number of MPI servers'''
        self.outputms = 'baseline1.mms'
        partition(self.vis,
                  outputvis=self.outputms,
                  separationaxis='baseline',
                  flagbackup=False)

        # The separation axis should be written to the output MMS
        sepaxis = ph.axisType(self.outputms)
        self.assertEqual(sepaxis, 'baseline',
                         'Partition did not write AxisType correctly in MMS')
Exemple #12
0
    def test_channels3(self):
        '''partition: verify spw sub-table consolidation'''
        partition(vis=self.msfile, outputvis=self.mmsfile, spw='3,5:10~19,7,9,11,13,15',
                    createmms=True,separationaxis='spw', flagbackup=False, datacolumn='data')       
                             
        self.assertTrue(os.path.exists(self.mmsfile))
        
        # spw=5 should be spw=1 after consolidation, with 10 channels
        ret = th.verifyMS(self.mmsfile, 7, 10, 1, ignoreflags=True)
        self.assertTrue(ret[0],ret[1])

        # The separation axis should be written to the output MMS
        sepaxis = ph.axisType(self.mmsfile)
        self.assertEqual(sepaxis, 'spw', 'Partition did not write AxisType correctly in MMS')
Exemple #13
0
    def test_channels_mms1(self):
        '''mstransform: create MMS with spw separation and channel selections'''
        self.outputms = "testmms1.mms"
        mstransform(vis=self.vis, outputvis=self.outputms, spw='0~4,5:1~10',createmms=True,
                    separationaxis='spw',disableparallel=True)

        self.assertTrue(os.path.exists(self.outputms))

        # It should create 6 subMS, with spw=0~5
        # spw=5 should have only 10 channels
        ret = th.verifyMS(self.outputms, 6, 10, 5,ignoreflags=True)
        self.assertTrue(ret[0],ret[1])

        # The separation axis should be written to the output MMS
        sepaxis = ph.axisType(self.outputms)
        self.assertEqual(sepaxis, 'spw', 'AxisType is not correctly written to output MMS')
    def test_mms3(self):
        '''test_mms3: Create MMS with separationaxis=spw and lazy=True'''
        myasdmname = 'uid___A002_X71e4ae_X317_short'
        themsname = myasdmname + ".ms"

        importasdm(myasdmname,
                   createmms=True,
                   lazy=True,
                   scans='1,2',
                   separationaxis='spw',
                   flagbackup=False,
                   process_flags=False)
        self.assertTrue(ParallelDataHelper.isParallelMS(themsname),
                        'Output is not a Multi-MS')
        self.assertEqual(ph.axisType(themsname), 'spw',
                         'Separation axis of MMS should be spw')
Exemple #15
0
 def test_timespan_spw_axis(self):
     '''mstransform: timeaverage=True, timespan=scan, separationaxis=spw'''
     self.createMMS(self.vis, axis='spw',spws='1,3')
     self.outputms = "spanscan_spw.mms"
     mstransform(vis=self.testmms, outputvis=self.outputms, datacolumn='data',timeaverage=True, 
                 timebin='100s',timespan='scan')
     self.assertTrue(ParallelDataHelper.isParallelMS(self.outputms),'Output should be an MMS')
     self.assertEqual(ph.axisType(self.outputms),'spw')
     
     mymsmd = msmdtool()
     mymsmd.open(self.outputms)
     t30 = mymsmd.exposuretime(30)['value']
     t31 = mymsmd.exposuretime(31)['value']
     mymsmd.close()
     self.assertEqual(t30, 100)
     self.assertEqual(t31, 79)
Exemple #16
0
    def test_channels_mms3(self):
        '''mstransform: create MMS with scan separation and channel selections'''
        self.outputms = "testmms3.mms"
        mstransform(vis=self.vis, outputvis=self.outputms, spw='0:0~10,1:60~63',createmms=True,
                    separationaxis='scan', disableparallel=True)
        self.assertTrue(os.path.exists(self.outputms))

        # It should create 2 subMS, with spw=0~1
        # spw=0 has 11 channels, spw=1 has 4 channels
        ret = th.verifyMS(self.outputms, 2, 11, 0, ignoreflags=True)
        self.assertTrue(ret[0],ret[1])
        ret = th.verifyMS(self.outputms, 2, 4, 1, ignoreflags=True)
        self.assertTrue(ret[0],ret[1])

        # The separation axis should be written to the output MMS
        sepaxis = ph.axisType(self.outputms)
        self.assertEqual(sepaxis, 'scan', 'AxisType is not correctly written to output MMS')
Exemple #17
0
    def test_scan_spw(self):
        '''Partition: separationaxis=scan with spw selection'''
        partition(vis=self.msfile,
                  outputvis=self.mmsfile,
                  separationaxis='scan',
                  spw='1~4,10,11',
                  flagbackup=False,
                  datacolumn='data')

        self.assertTrue(os.path.exists(self.mmsfile),
                        'MMS was not created for this test')
        self.assertTrue(
            os.path.exists(self.msfile),
            'Make sure the input MS is not deleted inside the task')

        # Take the dictionary and compare with original MS
        thisdict = listpartition(vis=self.mmsfile, createdict=True)

        # Dictionary with selection to compare with original MS
        mysel = {'spw': '1~4,10,11'}

        # Compare nrows of all scans in selection
        slist = ph.getMMSScans(thisdict)
        for s in slist:
            mmsN = ph.getMMSScanNrows(thisdict, s)
            msN = ph.getScanNrows(self.msfile, s, selection=mysel)
            self.assertEqual(
                mmsN, msN,
                'Nrows in scan=%s differs: mms_nrows=%s <--> ms_nrows=%s' %
                (s, mmsN, msN))

        # The comparison should be
        # ms_spw = 1 --> mms_spw = 0
        # ms_spw = 2 --> mms_spw = 1, etc.
        # Check that MMS spw IDs have been re-indexed properly
        indexed_ids = range(6)
        for s in slist:
            mms_spw = ph.getSpwIds(self.mmsfile, s)
            self.assertEqual(mms_spw, indexed_ids,
                             'spw IDs were not properly re-indexed')

        # The separation axis should be written to the output MMS
        sepaxis = ph.axisType(self.mmsfile)
        self.assertEqual(sepaxis, 'scan',
                         'Partition did not write AxisType correctly in MMS')
Exemple #18
0
    def test_sepaxis(self):
        '''Partition: separationaxis=auto'''
        partition(vis=self.msfile,
                  outputvis=self.mmsfile,
                  spw='0~11',
                  separationaxis='auto',
                  flagbackup=False,
                  disableparallel=True,
                  datacolumn='data')
        #        partition(vis=self.msfile, outputvis=self.mmsfile,separationaxis='auto')

        self.assertTrue(os.path.exists(self.mmsfile),
                        'MMS was not created for this test')

        # Dictionary with selection to compare with original MS
        mysel = {'spw': '0~11'}

        # Take the dictionary and compare with original MS
        thisdict = listpartition(vis=self.mmsfile, createdict=True)

        # Compare nrows of all scans in selection
        slist = ph.getMMSScans(thisdict)
        self.assertEqual(slist.__len__(), 2)
        for s in slist:
            mmsN = ph.getMMSScanNrows(thisdict, s)
            msN = ph.getScanNrows(self.msfile, s, selection=mysel)
            #            msN = ph.getScanNrows(self.msfile, s)
            self.assertEqual(
                mmsN, msN,
                'Nrows in scan=%s differs: mms_nrows=%s <--> ms_nrows=%s' %
                (s, mmsN, msN))

        # Compare spw IDs
        for s in slist:
            mms_spw = ph.getSpwIds(self.mmsfile, s)
            ms_spw = ph.getSpwIds(self.msfile, s, selection=mysel)
            #            ms_spw = ph.getSpwIds(self.msfile, s)
            self.assertEqual(mms_spw, ms_spw, 'list of spws in scan=%s differs: '\
                             'mms_spw=%s <--> ms_spw=%s' %(s, mms_spw, ms_spw))

        # The separation axis should be written to the output MMS
        sepaxis = ph.axisType(self.mmsfile)
        self.assertEqual(sepaxis, 'scan,spw',
                         'Partition did not write AxisType correctly in MMS')
Exemple #19
0
    def test_numsubms(self):
        '''Partition: small numsubms value'''
        # There are 16 spws; we want only 6 sub-MSs.
        partition(vis=self.msfile,
                  outputvis=self.mmsfile,
                  separationaxis='spw',
                  numsubms=6,
                  flagbackup=False,
                  disableparallel=True,
                  datacolumn='corrected')

        self.assertTrue(os.path.exists(self.mmsfile),
                        'MMS was not created for this test')

        # Take the dictionary and compare with original MS
        thisdict = listpartition(vis=self.mmsfile, createdict=True)

        # Check the number of sub-MSs
        mmslist = []
        klist = thisdict.keys()
        for kk in klist:
            mmslist.append(thisdict[kk]['MS'])

        nsubms = mmslist.__len__()
        self.assertEqual(nsubms, 6, 'There should be only 6 sub-MSs')

        # Check that spw list is the same in MS and MMS
        spwlist = ph.getMMSSpwIds(thisdict)

        # Reference list of scans in MS
        slist = ph.getScanList(self.msfile)
        setlist = set([])
        for s in slist:
            ms_spw = ph.getSpwIds(self.msfile, s)
            for a in ms_spw:
                setlist.add(a)

        self.assertEqual(list(setlist), spwlist)

        # The separation axis should be written to the output MMS
        sepaxis = ph.axisType(self.mmsfile)
        self.assertEqual(sepaxis, 'spw',
                         'Partition did not write AxisType correctly in MMS')
Exemple #20
0
    def test_numsubms(self):
        """Partition: small numsubms value"""
        # There are 16 spws; we want only 6 sub-MSs.
        partition(
            vis=self.msfile,
            outputvis=self.mmsfile,
            separationaxis="spw",
            numsubms=6,
            flagbackup=False,
            disableparallel=True,
            datacolumn="corrected",
        )

        self.assertTrue(os.path.exists(self.mmsfile), "MMS was not created for this test")

        # Take the dictionary and compare with original MS
        thisdict = listpartition(vis=self.mmsfile, createdict=True)

        # Check the number of sub-MSs
        mmslist = []
        klist = thisdict.keys()
        for kk in klist:
            mmslist.append(thisdict[kk]["MS"])

        nsubms = mmslist.__len__()
        self.assertEqual(nsubms, 6, "There should be only 6 sub-MSs")

        # Check that spw list is the same in MS and MMS
        spwlist = ph.getMMSSpwIds(thisdict)

        # Reference list of scans in MS
        slist = ph.getScanList(self.msfile)
        setlist = set([])
        for s in slist:
            ms_spw = ph.getSpwIds(self.msfile, s)
            for a in ms_spw:
                setlist.add(a)

        self.assertEqual(list(setlist), spwlist)

        # The separation axis should be written to the output MMS
        sepaxis = ph.axisType(self.mmsfile)
        self.assertEqual(sepaxis, "spw", "Partition did not write AxisType correctly in MMS")
Exemple #21
0
    def test_sepaxis(self):
        """Partition: separationaxis=auto"""
        partition(
            vis=self.msfile,
            outputvis=self.mmsfile,
            spw="0~11",
            separationaxis="auto",
            flagbackup=False,
            disableparallel=True,
            datacolumn="data",
        )
        #        partition(vis=self.msfile, outputvis=self.mmsfile,separationaxis='auto')

        self.assertTrue(os.path.exists(self.mmsfile), "MMS was not created for this test")

        # Dictionary with selection to compare with original MS
        mysel = {"spw": "0~11"}

        # Take the dictionary and compare with original MS
        thisdict = listpartition(vis=self.mmsfile, createdict=True)

        # Compare nrows of all scans in selection
        slist = ph.getMMSScans(thisdict)
        self.assertEqual(slist.__len__(), 2)
        for s in slist:
            mmsN = ph.getMMSScanNrows(thisdict, s)
            msN = ph.getScanNrows(self.msfile, s, selection=mysel)
            #            msN = ph.getScanNrows(self.msfile, s)
            self.assertEqual(mmsN, msN, "Nrows in scan=%s differs: mms_nrows=%s <--> ms_nrows=%s" % (s, mmsN, msN))

        # Compare spw IDs
        for s in slist:
            mms_spw = ph.getSpwIds(self.mmsfile, s)
            ms_spw = ph.getSpwIds(self.msfile, s, selection=mysel)
            #            ms_spw = ph.getSpwIds(self.msfile, s)
            self.assertEqual(
                mms_spw, ms_spw, "list of spws in scan=%s differs: " "mms_spw=%s <--> ms_spw=%s" % (s, mms_spw, ms_spw)
            )

        # The separation axis should be written to the output MMS
        sepaxis = ph.axisType(self.mmsfile)
        self.assertEqual(sepaxis, "scan,spw", "Partition did not write AxisType correctly in MMS")
Exemple #22
0
    def test_channels3(self):
        '''partition: verify spw sub-table consolidation'''
        partition(vis=self.msfile,
                  outputvis=self.mmsfile,
                  spw='3,5:10~19,7,9,11,13,15',
                  createmms=True,
                  separationaxis='spw',
                  flagbackup=False,
                  datacolumn='data')

        self.assertTrue(os.path.exists(self.mmsfile))

        # spw=5 should be spw=1 after consolidation, with 10 channels
        ret = th.verifyMS(self.mmsfile, 7, 10, 1, ignoreflags=True)
        self.assertTrue(ret[0], ret[1])

        # The separation axis should be written to the output MMS
        sepaxis = ph.axisType(self.mmsfile)
        self.assertEqual(sepaxis, 'spw',
                         'Partition did not write AxisType correctly in MMS')
Exemple #23
0
    def test_channels_mms2(self):
        '''mstransform: create MMS with spw/scan separation and channel selections'''
        self.outputms = "testmms2.mms"
        mstransform(vis=self.vis, outputvis=self.outputms, spw='0:0~10,1:60~63',createmms=True,
                    separationaxis='auto', disableparallel=True)

        self.assertTrue(os.path.exists(self.outputms))

        # It should create 4 subMS, with spw=0~1
        # spw=0 has 11 channels, spw=1 has 4 channels
        ret = th.verifyMS(self.outputms, 2, 11, 0, ignoreflags=True)
        self.assertTrue(ret[0],ret[1])
        ret = th.verifyMS(self.outputms, 2, 4, 1, ignoreflags=True)
        self.assertTrue(ret[0],ret[1])

        # Verify that some sub-tables are properly re-indexed.
        spw_col = th.getVarCol(self.outputms+'/DATA_DESCRIPTION', 'SPECTRAL_WINDOW_ID')
        self.assertEqual(spw_col.keys().__len__(), 2, 'Wrong number of rows in DD table')
        self.assertEqual(spw_col['r1'][0], 0,'Error re-indexing DATA_DESCRIPTION table')
        self.assertEqual(spw_col['r2'][0], 1,'Error re-indexing DATA_DESCRIPTION table')

        # The separation axis should be written to the output MMS
        sepaxis = ph.axisType(self.outputms)
        self.assertEqual(sepaxis, 'scan,spw', 'AxisType is not correctly written to output MMS')
Exemple #24
0
    def test_default(self):
        '''Partition: create an MMS with default values in parallel'''

        # First split off one scan to run the test faster
        split(vis=self.msfile,
              outputvis='split30.ms',
              datacolumn='DATA',
              scan='30')
        msfile = 'split30.ms'

        partition(vis=msfile, outputvis=self.mmsfile)

        self.assertTrue(os.path.exists(self.mmsfile),
                        'MMS was not created for this test')

        # Gather several metadata information
        # for the MS
        mdlocal1 = msmdtool()
        mdlocal1.open(msfile)
        ms_rows = mdlocal1.nrows()
        ms_nscans = mdlocal1.nscans()
        ms_nspws = mdlocal1.nspw()
        ms_scans = mdlocal1.scannumbers()
        mdlocal1.close()

        # for the MMS
        mdlocal2 = msmdtool()
        mdlocal2.open(self.mmsfile)
        mms_rows = mdlocal2.nrows()
        mms_nscans = mdlocal2.nscans()
        mms_nspws = mdlocal2.nspw()
        mms_scans = mdlocal2.scannumbers()
        mdlocal2.close()

        # Compare the number of rows
        self.assertEqual(ms_rows, mms_rows,
                         'Compare total number of rows in MS and MMS')
        self.assertEqual(ms_nscans, mms_nscans, 'Compare number of scans')
        self.assertEqual(ms_nspws, mms_nspws, 'Compare number of spws')

        # Compare the scans
        self.assertEqual(ms_scans.all(), mms_scans.all(),
                         'Compare all scan IDs')

        try:
            mdlocal1.open(msfile)
            mdlocal2.open(self.mmsfile)

            # Compare the spws
            for i in ms_scans:
                msi = mdlocal1.spwsforscan(i)
                mmsi = mdlocal2.spwsforscan(i)
                self.assertEqual(msi.all(), mmsi.all(),
                                 'Compare spw Ids for a scan')
        finally:
            mdlocal1.close()
            mdlocal2.close()

        # Sort the output MSs so that they can be compared
        myms = mstool()

        myms.open(msfile)
        myms.sort('ms_sorted.ms', [
            'OBSERVATION_ID', 'ARRAY_ID', 'SCAN_NUMBER', 'FIELD_ID',
            'DATA_DESC_ID', 'ANTENNA1', 'ANTENNA2', 'TIME'
        ])
        myms.done()

        myms.open(self.mmsfile)
        myms.sort('mms_sorted.ms', [
            'OBSERVATION_ID', 'ARRAY_ID', 'SCAN_NUMBER', 'FIELD_ID',
            'DATA_DESC_ID', 'ANTENNA1', 'ANTENNA2', 'TIME'
        ])
        myms.done()

        # Ignore WEIGHT_SPECTRUM and SIGMA_SPECTRUM, which are empty columns
        self.assertTrue(
            th.compTables('ms_sorted.ms', 'mms_sorted.ms', [
                'FLAG', 'FLAG_CATEGORY', 'TIME_CENTROID', 'WEIGHT_SPECTRUM',
                'SIGMA_SPECTRUM', 'DATA'
            ]))

        # Compare the DATA column
        self.assertTrue(
            th.compVarColTables('ms_sorted.ms', 'mms_sorted.ms', 'DATA'))

        # The separation axis should be written to the output MMS
        sepaxis = ph.axisType(self.mmsfile)
        self.assertEqual(sepaxis, 'scan,spw',
                         'Partition did not write AxisType correctly in MMS')
    def test_mms4(self):
        '''test_mms4: Create MMS, lazy=True, with separationaxis=scan and scans selection in ASDM'''
        retValue = {'success': True, 'msgs': "", 'error_msgs': ''}

        myasdmname = 'uid___A002_X72bc38_X000'
        themsname = myasdmname + ".ms"

        # only the first 3 scans to save time
        importasdm(myasdmname,
                   vis=themsname,
                   lazy=True,
                   scans='0:1~3',
                   createmms=True,
                   separationaxis='scan',
                   process_flags=False)
        self.assertTrue(ParallelDataHelper.isParallelMS(themsname),
                        'Output is not a Multi-MS')
        self.assertEqual(ph.axisType(themsname), 'scan',
                         'Separation axis of MMS should be scan')
        print myname, ": Success! Now checking output ..."
        mscomponents = set([
            "ANTENNA/table.dat", "DATA_DESCRIPTION/table.dat",
            "FEED/table.dat", "FIELD/table.dat", "FLAG_CMD/table.dat",
            "HISTORY/table.dat", "OBSERVATION/table.dat", "POINTING/table.dat",
            "POLARIZATION/table.dat", "PROCESSOR/table.dat",
            "SOURCE/table.dat", "SPECTRAL_WINDOW/table.dat", "STATE/table.dat",
            "SYSCAL/table.dat", "ANTENNA/table.f0",
            "DATA_DESCRIPTION/table.f0", "FEED/table.f0", "FIELD/table.f0",
            "FLAG_CMD/table.f0", "HISTORY/table.f0", "OBSERVATION/table.f0",
            "POINTING/table.f0", "POLARIZATION/table.f0", "PROCESSOR/table.f0",
            "SOURCE/table.f0", "SPECTRAL_WINDOW/table.f0", "STATE/table.f0",
            "SYSCAL/table.f0"
        ])
        for name in mscomponents:
            if not os.access(themsname + "/" + name, os.F_OK):
                print myname, ": Error  ", themsname + "/" + name, "doesn't exist ..."
                retValue['success'] = False
                retValue['error_msgs'] = retValue[
                    'error_msgs'] + themsname + '/' + name + ' does not exist'
            else:
                print myname, ": ", name, "present."
        print myname, ": MS exists. All tables present. Try opening as MS ..."
        try:
            mslocal.open(themsname)
        except:
            print myname, ": Error  Cannot open MS table", themsname
            retValue['success'] = False
            retValue['error_msgs'] = retValue[
                'error_msgs'] + 'Cannot open MS table ' + themsname
        else:
            mslocal.close()
            print myname, ": OK. Checking tables in detail ..."

            importasdm(asdm=myasdmname,
                       vis='reference.ms',
                       lazy=False,
                       overwrite=True,
                       scans='0:1~3',
                       createmms=True,
                       separationaxis='scan',
                       process_flags=False)

            if (os.path.exists('reference.ms')):
                retValue['success'] = th.checkwithtaql(
                    "select from [select from reference.ms orderby TIME, DATA_DESC_ID, ANTENNA1, ANTENNA2 ] t1, [select from "
                    + themsname +
                    " orderby TIME, DATA_DESC_ID, ANTENNA1, ANTENNA2 ] t2 where (not all(near(t1.DATA,t2.DATA, 1.e-06)))"
                ) == 0
                if not retValue['success']:
                    print "ERROR: DATA does not agree with reference."
                else:
                    print "DATA columns agree."
                retValueTmp = th.checkwithtaql(
                    "select from [select from reference.ms orderby TIME, DATA_DESC_ID, ANTENNA1, ANTENNA2 ] t1, [select from "
                    + themsname +
                    " orderby TIME, DATA_DESC_ID, ANTENNA1, ANTENNA2 ] t2 where (not all(t1.FLAG==t2.FLAG)) "
                ) == 0
                if not retValueTmp:
                    print "ERROR: FLAG does not agree with reference."
                else:
                    print "FLAG columns agree."

                retValue['success'] = retValue['success'] and retValueTmp

                for subtname in [
                        "ANTENNA", "DATA_DESCRIPTION", "FEED", "FIELD",
                        "FLAG_CMD", "OBSERVATION", "POINTING", "POLARIZATION",
                        "PROCESSOR", "SOURCE", "SPECTRAL_WINDOW", "STATE",
                        "SYSCAL"
                ]:

                    print "\n*** Subtable ", subtname
                    excllist = []
                    if subtname == 'SOURCE':
                        excllist = [
                            'POSITION', 'TRANSITION', 'REST_FREQUENCY',
                            'SYSVEL'
                        ]
                    if subtname == 'SYSCAL':
                        excllist = ['TANT_SPECTRUM', 'TANT_TSYS_SPECTRUM']
                    if subtname == 'SPECTRAL_WINDOW':
                        excllist = [
                            'CHAN_FREQ', 'CHAN_WIDTH', 'EFFECTIVE_BW',
                            'RESOLUTION'
                        ]
                        for colname in excllist:
                            retValue['success'] = th.compVarColTables(
                                'reference.ms/SPECTRAL_WINDOW',
                                themsname + '/SPECTRAL_WINDOW', colname,
                                0.01) and retValue['success']

                    try:
                        retValue['success'] = th.compTables(
                            'reference.ms/' + subtname, themsname + '/' +
                            subtname, excllist, 0.01) and retValue['success']
                    except:
                        retValue['success'] = False
                        print "ERROR for table ", subtname

        print retValue
        self.assertTrue(retValue['success'], retValue['error_msgs'])
Exemple #26
0
def listpartition(vis=None, createdict=None, listfile=None):
    """List information about an MMS data set in the logger:

       Keyword arguments:
       vis -- Name of multi-MS or normal MS.
               default: none. example: vis='uidA002.mms'
       createdict -- Create and return a dictionary with information about
                     the sub-MSs.
               default: False
       listfile -- save the output to a file
             default: ''. Example: listfile="mylist.txt"
             
        The task also returns a dictionary with scan summary information
        for each sub-MS. 
                      

       """

    casalog.origin('listpartition')

    mslocal = casac.ms()
    mslocal1 = casac.ms()

    try:
        if (type(vis) == str) & os.path.exists(vis):
            mslocal.open(thems=vis)
        else:
            raise Exception, \
                'Visibility data set not found - please verify the name'

        # Check output filename existence
        if listfile != '':
            if (type(listfile) == str) & os.path.exists(listfile):
                raise Exception, 'Output file \'%s\' already exists' % listfile

            casalog.post('Will save output to \'%s\'' % listfile)
            ffout = open(listfile, 'w')

        # Is it a multi-MS?
        ismms = mslocal.ismultims()

        # List of MSs to process
        mslist = []

        # It is a multi-MS
        if ismms:
            mslist = mslocal.getreferencedtables()
            mslist.sort()
            sname = 'Sub-MS'

            # Get the AxisType of the MMS
            axis = ph.axisType(vis)
            if axis == '':
                axis = 'unknown'

            casalog.post('This is a multi-MS with separation axis = ' + axis)

        else:
            mslist.append(vis)
            sname = 'MS'

        # Close top MS
        mslocal.close()

        # Get a consolidated dictionary with scan, spw, channel information
        # of the list of subMSs. It adds the nrows of all sub-scans of a scan.
        try:
            outdict = {}
            outdict = ph.getScanSpwSummary(mslist)
        except Exception, instance:
            casalog.post('%s' % instance, 'ERROR')

        # Now loop through the dictionary to print the information
        if outdict.keys() == []:
            casalog.post('Error in processing dictionaries', 'ERROR')

        indices = outdict.keys()
        indices.sort()

        counter = 0
        for index in indices:

            # Get data
            MS = outdict[index]['MS']
            SIZE = outdict[index]['size']
            SCAN = outdict[index]['scanId']

            # Sort scans for more optimal printing
            # Print information per scan
            firstscan = True
            skeys = SCAN.keys()
            skeys.sort()
            for myscan in skeys:
                SPW = outdict[index]['scanId'][myscan]['spwIds']
                NCHAN = outdict[index]['scanId'][myscan]['nchans']
                NROWS = outdict[index]['scanId'][myscan]['nrows']

                # Get maximum widths
                smxw = getWidth(outdict, 'spw')
                cmxw = getWidth(outdict, 'channel')

                # Create format
                fhdr = '%-'+str(len(MS)+2)+'s' + '%-6s' + '%-'+str(smxw+2)+'s' + \
                        '%-'+str(cmxw+2)+'s' + '%-8s' + '%-6s'

                # Print header
                text = ''
                if counter == 0:
                    text = text + fhdr % (sname, 'Scan', 'Spw', 'Nchan',
                                          'Nrows', 'Size')
                    text = text + '\n'
                counter += 1

                # Print first scan
                if firstscan:
                    text = text + fhdr % (MS, myscan, SPW, NCHAN, NROWS, SIZE)
                else:
                    text = text + fhdr % ('', myscan, SPW, NCHAN, NROWS, '')

                firstscan = False

                # Print to a file
                if listfile != '':
                    print >> ffout, text
                else:
                    # Print to the logger
                    casalog.post(text)

        if listfile != '':
            ffout.close()

        # Return the scan dictionary
        if createdict:
            return outdict

        return {}
Exemple #27
0
def listpartition(vis=None, createdict=None, listfile=None):
    
    """List information about an MMS data set in the logger:

       Keyword arguments:
       vis -- Name of multi-MS or normal MS.
               default: none. example: vis='uidA002.mms'
       createdict -- Create and return a dictionary with information about
                     the sub-MSs.
               default: False
       listfile -- save the output to a file
             default: ''. Example: listfile="mylist.txt"
             
        The task also returns a dictionary with scan summary information
        for each sub-MS. 
                      

       """

    casalog.origin('listpartition')

    mslocal = casac.ms()
    mslocal1 = casac.ms()
            

    try:
        if (type(vis) == str) & os.path.exists(vis):
            mslocal.open(thems=vis)
        else:
            raise Exception, \
                'Visibility data set not found - please verify the name'

        # Check output filename existence 
        if listfile != '':
            if (type(listfile) == str) & os.path.exists(listfile):
                raise Exception, 'Output file \'%s\' already exists'%listfile
            
            casalog.post('Will save output to \'%s\''%listfile)
            ffout = open(listfile, 'w')
            
                
        # Is it a multi-MS?
        ismms = mslocal.ismultims()
        
        # List of MSs to process
        mslist = []
        
        # It is a multi-MS
        if ismms:
            mslist = mslocal.getreferencedtables()
            mslist.sort()
            sname = 'Sub-MS'
            
            # Get the AxisType of the MMS
            axis = ph.axisType(vis)
            if axis == '':
                axis = 'unknown'
                
            casalog.post('This is a multi-MS with separation axis = '+axis)

        else:
            mslist.append(vis)
            sname = 'MS'
            
        # Close top MS
        mslocal.close()

        # Get a consolidated dictionary with scan, spw, channel information
        # of the list of subMSs. It adds the nrows of all sub-scans of a scan.
        try:
            outdict = {}
            outdict = ph.getScanSpwSummary(mslist) 
        except Exception, instance:
            casalog.post('%s'%instance,'ERROR')

        # Now loop through the dictionary to print the information
        if outdict.keys() == []:
            casalog.post('Error in processing dictionaries','ERROR')
        
        indices = outdict.keys()
        indices.sort()
            
        counter = 0
        for index in indices:
            
            # Get data
            MS = outdict[index]['MS']            
            SIZE = outdict[index]['size']
            SCAN = outdict[index]['scanId']
                        
            # Sort scans for more optimal printing
            # Print information per scan
            firstscan = True
            skeys = SCAN.keys()
            skeys.sort()
            for myscan in skeys:
                SPW = outdict[index]['scanId'][myscan]['spwIds']
                NCHAN = outdict[index]['scanId'][myscan]['nchans']
                NROWS = outdict[index]['scanId'][myscan]['nrows']
                
                # Get maximum widths
                smxw = getWidth(outdict, 'spw')
                cmxw = getWidth(outdict, 'channel')
                
                # Create format
                fhdr = '%-'+str(len(MS)+2)+'s' + '%-6s' + '%-'+str(smxw+2)+'s' + \
                        '%-'+str(cmxw+2)+'s' + '%-8s' + '%-6s'
    
                
                # Print header
                text = ''
                if counter == 0:
                    text = text + fhdr % (sname,'Scan','Spw','Nchan','Nrows','Size')  
                    text = text + '\n'                  
                counter += 1
                
                # Print first scan
                if firstscan:
                    text = text + fhdr % (MS, myscan, SPW, NCHAN, NROWS, SIZE)   
                else:
                    text = text + fhdr % ('', myscan, SPW, NCHAN, NROWS, '')
                
                firstscan = False            

                # Print to a file
                if listfile != '':
                    print >> ffout, text
                else:
                    # Print to the logger
                    casalog.post(text)
                                
                
        if listfile != '':    
            ffout.close()
                                        
     
        # Return the scan dictionary
        if createdict:
            return outdict
        
        return {}
Exemple #28
0
    def test_default(self):
        '''Partition: create an MMS with default values in parallel'''
        
        # First split off one scan to run the test faster
        split(vis=self.msfile, outputvis='split30.ms', datacolumn='DATA', scan='30')
        msfile = 'split30.ms'

        partition(vis=msfile, outputvis=self.mmsfile)
        
        self.assertTrue(os.path.exists(self.mmsfile), 'MMS was not created for this test')
        
        # Gather several metadata information
        # for the MS
        mdlocal1 = msmdtool()
        mdlocal1.open(msfile)
        ms_rows = mdlocal1.nrows()
        ms_nscans = mdlocal1.nscans()
        ms_nspws = mdlocal1.nspw()
        ms_scans = mdlocal1.scannumbers()
        mdlocal1.close()        
          
        # for the MMS
        mdlocal2 = msmdtool()
        mdlocal2.open(self.mmsfile)
        mms_rows = mdlocal2.nrows()
        mms_nscans = mdlocal2.nscans()
        mms_nspws = mdlocal2.nspw()
        mms_scans = mdlocal2.scannumbers()
        mdlocal2.close()        
          
        # Compare the number of rows
        self.assertEqual(ms_rows, mms_rows, 'Compare total number of rows in MS and MMS')
        self.assertEqual(ms_nscans, mms_nscans, 'Compare number of scans')
        self.assertEqual(ms_nspws, mms_nspws, 'Compare number of spws')
          
        # Compare the scans
        self.assertEqual(ms_scans.all(), mms_scans.all(), 'Compare all scan IDs')
  
        try:
            mdlocal1.open(msfile)
            mdlocal2.open(self.mmsfile)
          
            # Compare the spws
            for i in ms_scans:                
                msi = mdlocal1.spwsforscan(i)
                mmsi = mdlocal2.spwsforscan(i)
                self.assertEqual(msi.all(), mmsi.all(), 'Compare spw Ids for a scan')
        finally:          
            mdlocal1.close()
            mdlocal2.close()               

        # Sort the output MSs so that they can be compared
        myms = mstool()
        
        myms.open(msfile)
        myms.sort('ms_sorted.ms',['OBSERVATION_ID','ARRAY_ID','SCAN_NUMBER','FIELD_ID','DATA_DESC_ID','ANTENNA1','ANTENNA2','TIME'])
        myms.done()
        
        myms.open(self.mmsfile)
        myms.sort('mms_sorted.ms',['OBSERVATION_ID','ARRAY_ID','SCAN_NUMBER','FIELD_ID','DATA_DESC_ID','ANTENNA1','ANTENNA2','TIME'])
        myms.done()

        # Ignore WEIGHT_SPECTRUM and SIGMA_SPECTRUM, which are empty columns
        self.assertTrue(th.compTables('ms_sorted.ms', 'mms_sorted.ms', 
                                      ['FLAG','FLAG_CATEGORY','TIME_CENTROID',
                                       'WEIGHT_SPECTRUM','SIGMA_SPECTRUM','DATA']))

        # Compare the DATA column
        self.assertTrue(th.compVarColTables('ms_sorted.ms', 'mms_sorted.ms','DATA'))
        
        # The separation axis should be written to the output MMS
        sepaxis = ph.axisType(self.mmsfile)
        self.assertEqual(sepaxis, 'scan,spw', 'Partition did not write AxisType correctly in MMS')
Exemple #29
0
    def test_default(self):
        """Partition: create an MMS with default values in parallel"""

        # First split off one scan to run the test faster
        split(vis=self.msfile, outputvis="split30.ms", datacolumn="DATA", scan="30")
        msfile = "split30.ms"

        partition(vis=msfile, outputvis=self.mmsfile)

        self.assertTrue(os.path.exists(self.mmsfile), "MMS was not created for this test")

        # Gather several metadata information
        # for the MS
        mdlocal1 = msmdtool()
        mdlocal1.open(msfile)
        ms_rows = mdlocal1.nrows()
        ms_nscans = mdlocal1.nscans()
        ms_nspws = mdlocal1.nspw()
        ms_scans = mdlocal1.scannumbers()
        mdlocal1.close()

        # for the MMS
        mdlocal2 = msmdtool()
        mdlocal2.open(self.mmsfile)
        mms_rows = mdlocal2.nrows()
        mms_nscans = mdlocal2.nscans()
        mms_nspws = mdlocal2.nspw()
        mms_scans = mdlocal2.scannumbers()
        mdlocal2.close()

        # Compare the number of rows
        self.assertEqual(ms_rows, mms_rows, "Compare total number of rows in MS and MMS")
        self.assertEqual(ms_nscans, mms_nscans, "Compare number of scans")
        self.assertEqual(ms_nspws, mms_nspws, "Compare number of spws")

        # Compare the scans
        self.assertEqual(ms_scans.all(), mms_scans.all(), "Compare all scan IDs")

        try:
            mdlocal1.open(msfile)
            mdlocal2.open(self.mmsfile)

            # Compare the spws
            for i in ms_scans:
                msi = mdlocal1.spwsforscan(i)
                mmsi = mdlocal2.spwsforscan(i)
                self.assertEqual(msi.all(), mmsi.all(), "Compare spw Ids for a scan")
        finally:
            mdlocal1.close()
            mdlocal2.close()

        # Sort the output MSs so that they can be compared
        myms = mstool()

        myms.open(msfile)
        myms.sort(
            "ms_sorted.ms",
            ["OBSERVATION_ID", "ARRAY_ID", "SCAN_NUMBER", "FIELD_ID", "DATA_DESC_ID", "ANTENNA1", "ANTENNA2", "TIME"],
        )
        myms.done()

        myms.open(self.mmsfile)
        myms.sort(
            "mms_sorted.ms",
            ["OBSERVATION_ID", "ARRAY_ID", "SCAN_NUMBER", "FIELD_ID", "DATA_DESC_ID", "ANTENNA1", "ANTENNA2", "TIME"],
        )
        myms.done()

        self.assertTrue(
            th.compTables(
                "ms_sorted.ms", "mms_sorted.ms", ["FLAG", "FLAG_CATEGORY", "TIME_CENTROID", "WEIGHT_SPECTRUM", "DATA"]
            )
        )

        # Compare the DATA column
        self.assertTrue(th.compVarColTables("ms_sorted.ms", "mms_sorted.ms", "DATA"))

        # The separation axis should be written to the output MMS
        sepaxis = ph.axisType(self.mmsfile)
        self.assertEqual(sepaxis, "scan,spw", "Partition did not write AxisType correctly in MMS")
    def test_mms4(self):
        '''test_mms4: Create MMS, lazy=True, with separationaxis=scan and scans selection in ASDM'''
        retValue = {'success': True, 'msgs': "", 'error_msgs': '' }    

        myasdmname = 'uid___A002_X72bc38_X000'
        themsname = myasdmname + ".ms"

        # only the first 3 scans to save time
        importasdm(myasdmname, vis=themsname, lazy=True, scans='0:1~3', createmms=True, separationaxis='scan',
                   process_flags=False) 
        self.assertTrue(ParallelDataHelper.isParallelMS(themsname), 'Output is not a Multi-MS')
        self.assertEqual(ph.axisType(themsname), 'scan', 'Separation axis of MMS should be scan')
        print myname, ": Success! Now checking output ..."
        mscomponents = set(["ANTENNA/table.dat",
                            "DATA_DESCRIPTION/table.dat",
                            "FEED/table.dat",
                            "FIELD/table.dat",
                            "FLAG_CMD/table.dat",
                            "HISTORY/table.dat",
                            "OBSERVATION/table.dat",
                            "POINTING/table.dat",
                            "POLARIZATION/table.dat",
                            "PROCESSOR/table.dat",
                            "SOURCE/table.dat",
                            "SPECTRAL_WINDOW/table.dat",
                            "STATE/table.dat",
                            "SYSCAL/table.dat",
                            "ANTENNA/table.f0",
                            "DATA_DESCRIPTION/table.f0",
                            "FEED/table.f0",
                            "FIELD/table.f0",
                            "FLAG_CMD/table.f0",
                            "HISTORY/table.f0",
                            "OBSERVATION/table.f0",
                            "POINTING/table.f0",
                            "POLARIZATION/table.f0",
                            "PROCESSOR/table.f0",
                            "SOURCE/table.f0",
                            "SPECTRAL_WINDOW/table.f0",
                            "STATE/table.f0",
                            "SYSCAL/table.f0"
                            ])
        for name in mscomponents:
            if not os.access(themsname+"/"+name, os.F_OK):
                print myname, ": Error  ", themsname+"/"+name, "doesn't exist ..."
                retValue['success']=False
                retValue['error_msgs']=retValue['error_msgs']+themsname+'/'+name+' does not exist'
            else:
                print myname, ": ", name, "present."
        print myname, ": MS exists. All tables present. Try opening as MS ..."
        try:
            mslocal.open(themsname)
        except:
            print myname, ": Error  Cannot open MS table", themsname
            retValue['success']=False
            retValue['error_msgs']=retValue['error_msgs']+'Cannot open MS table '+themsname
        else:
            mslocal.close()
            print myname, ": OK. Checking tables in detail ..."
    
            importasdm(asdm=myasdmname, vis='reference.ms', lazy=False, overwrite=True, scans='0:1~3',
                        createmms=True, separationaxis='scan', process_flags=False)

            if(os.path.exists('reference.ms')):
                retValue['success'] = th.checkwithtaql("select from [select from reference.ms orderby TIME, DATA_DESC_ID, ANTENNA1, ANTENNA2 ] t1, [select from "
                                                    +themsname+" orderby TIME, DATA_DESC_ID, ANTENNA1, ANTENNA2 ] t2 where (not all(near(t1.DATA,t2.DATA, 1.e-06)))") == 0
                if not retValue['success']:
                    print "ERROR: DATA does not agree with reference."
                else:
                    print "DATA columns agree."
                retValueTmp = th.checkwithtaql("select from [select from reference.ms orderby TIME, DATA_DESC_ID, ANTENNA1, ANTENNA2 ] t1, [select from "
                                            +themsname+" orderby TIME, DATA_DESC_ID, ANTENNA1, ANTENNA2 ] t2 where (not all(t1.FLAG==t2.FLAG)) ") == 0
                if not retValueTmp:
                    print "ERROR: FLAG does not agree with reference."
                else:
                    print "FLAG columns agree."

                retValue['success'] = retValue['success'] and retValueTmp

                for subtname in ["ANTENNA",
                                 "DATA_DESCRIPTION",
                                 "FEED",
                                 "FIELD",
                                 "FLAG_CMD",
                                 "OBSERVATION",
                                 "POINTING",
                                 "POLARIZATION",
                                 "PROCESSOR",
                                 "SOURCE",
                                 "SPECTRAL_WINDOW",
                                 "STATE",
                                 "SYSCAL"]:
                    
                    print "\n*** Subtable ",subtname
                    excllist = []
                    if subtname=='SOURCE':
                        excllist=['POSITION', 'TRANSITION', 'REST_FREQUENCY', 'SYSVEL']
                    if subtname=='SYSCAL':
                        excllist=['TANT_SPECTRUM', 'TANT_TSYS_SPECTRUM']
                    if subtname=='SPECTRAL_WINDOW':
                        excllist=['CHAN_FREQ', 'CHAN_WIDTH', 'EFFECTIVE_BW', 'RESOLUTION']
                        for colname in excllist: 
                            retValue['success'] = th.compVarColTables('reference.ms/SPECTRAL_WINDOW',
                                                                      themsname+'/SPECTRAL_WINDOW', colname, 0.01) and retValue['success']
                        
                    try:    
                        retValue['success'] = th.compTables('reference.ms/'+subtname,
                                                            themsname+'/'+subtname, excllist, 
                                                            0.01) and retValue['success']
                    except:
                        retValue['success'] = False
                        print "ERROR for table ", subtname
            
        print retValue    
        self.assertTrue(retValue['success'],retValue['error_msgs'])
Exemple #31
0
def pieflag(vis,
            field,          # data selection parameters
            refchanfile,
            fitorder_RR_LL,
            fitorder_RL_LR,
            scalethresh,
            SEFDfile,       # scalethresh parameter
            plotSEFD,
            dynamicflag,
            chunktime,      # dynamicflag parameters
            stdmax,
            maxoffset,
            staticflag,
            madmax,         # staticflag parameter
            binsamples,
            extendflag,
            boxtime,        # extendflag parameters
            boxthresh):

    #
    # Task pieflag
    #    Flags bad data by comparing with clean channels in bandpass-calibrated data.
    #
    #    Original reference: E. Middelberg, 2006, PASA, 23, 64
    #    Rewritten for use in CASA and updated to account for wideband
    #    and SEFD effects by Christopher A. Hales 2014.
    #
    #    Thanks to Kumar Golap, Justo Gonzalez, Jeff Kern, James Robnett,
    #    Urvashi Rau, Sanjay Bhatnagar, and of course Enno Middelberg
    #    for expert advice. Thanks to Emmanuel Momjian for providing
    #    Jansky VLA SEFD data for L and X bands (EVLA Memos 152 and 166)
    #    and to Bryan Butler for providing access to all other bands
    #    from the Jansky VLA Exposure Calculator.
    #
    #    Version 4.4 released 26 October 2016
    #    Tested with CASA 4.7.0 using Jansky VLA data
    #    Available at: http://github.com/chrishales/pieflag
    #
    #    Reference for this version:
    #    C. A. Hales, E. Middelberg, 2014, Astrophysics Source Code Library, 1408.014
    #    http://adsabs.harvard.edu/abs/2014ascl.soft08014H
    #
    
    startTime = time.time()
    casalog.origin('pieflag')
    casalog.post('--> pieflag version 4.4')
    
    if (not staticflag) and (not dynamicflag):
        casalog.post('*** ERROR: You need to select static or dynamic flagging.', 'ERROR')
        casalog.post('*** ERROR: Exiting pieflag.', 'ERROR')
        return
    
    ms.open(vis)
    vis=ms.name()
    ms.close()
    
    useMPI = MPIEnvironment.is_mpi_enabled
    if useMPI:
        if vis.lower().endswith('.ms'):
            useMPI=False
            casalog.post('--> MS will be processed in serial mode.')
        elif ph.axisType(vis) == 'baseline':
            # client is ID 0 and will not perform parallel processing, servers start from ID 1
            nthreads = MPIEnvironment.rank
            subms_path = vis+'/SUBMSS/'
            subms = filter(lambda x: os.path.isdir(os.path.join(subms_path, x)), os.listdir(subms_path))
            if len(subms) != nthreads:
                casalog.post('*** ERROR: Mismatch, MMS tailored for '+str(len(subms))+' engines but '+\
                                         'CASA session tailored for '+str(nthreads)+' engines.', 'ERROR')
                casalog.post('*** ERROR: Exiting pieflag.', 'ERROR')
                return
            
            server_list = MPIEnvironment.mpi_server_rank_list()
            casalog.post('--> Initializing MPI parallel cluster with '+str(nthreads)+' engines.')
            client = MPICommandClient()
            client.start_services()
            # do some detective work to find appropriate path to push to clients
            syspaths = sys.path
            n = 0
            for k in range(len(syspaths)):
                if os.path.isfile(syspaths[k]+'/mytasks.py'):
                    for line in open(syspaths[k]+'/mytasks.py','r'):
                        if re.search("task_location\['pieflag'\]",line):
                            if n==0:
                                n += 1
                                addpath = syspaths[k]
                            elif syspaths[k] != addpath:
                                n += 1
            
            if n == 1:
                casalog.filter('WARN')
                #client.set_log_level('WARN')
                client.push_command_request("casalog.filter('WARN')",True,server_list)
                client.push_command_request("sys.path.append('"+addpath+"')",True,server_list)
                client.push_command_request('from task_pieflag import pieflag_getflagstats',True,server_list)
                client.push_command_request('from task_pieflag import pieflag_flag',True,server_list)
                casalog.filter('INFO')
            else:
                if n == 0:
                    casalog.post('*** ERROR: pieflag mytasks.py installation not found in sys.path', 'ERROR')
                else:
                    casalog.post('*** ERROR: Ambiguity, sys.path contains more than 1 pieflag installation', 'ERROR')
                    casalog.post('***        (pieflag referenced in '+str(n)+' unique path/mytasks.py)', 'ERROR')
                
                casalog.post('*** ERROR: Exiting pieflag.', 'ERROR')
                return
            
            fcall1 = 'pieflag_getflagstats(vis,field,spw,npol,feedbasis)'
            fcall2 = 'pieflag_flag(vis,datacol,nthreads,field,vtbleLIST,inttime,nant,ddid,spw,refchan,nchan,npol,'+\
                     'feedbasis,fitorderLIST,sefdLIST,staticflag,madmax,binsamples,dynamicflag,chunktime,stdmax,'+\
                     'maxoffset,extendflag,boxtime,boxthresh)'
        else:
            casalog.post('*** ERROR: MMS is not partitioned by baseline. Cannot process.', 'ERROR')
            casalog.post('***        Use partition() to revert to MS then create baseline MMS.', 'ERROR')
            casalog.post('*** ERROR: Exiting pieflag.', 'ERROR')
            return
    else:
        if vis.lower().endswith('.mms'):
            casalog.post('*** ERROR: pieflag cannot handle MMS in non-MPI-enabled CASA session.', 'ERROR')
            casalog.post('*** ERROR: Exiting pieflag.', 'ERROR')
            return
        else:
            casalog.post('--> MS will be processed in serial mode.')
    
    tb.open(vis)
    if any('CORRECTED_DATA' in colnames for colnames in tb.colnames()):
        datacol='CORRECTED_DATA'
    else:
        datacol='DATA'
    
    tb.close()
    
    # load in reference channel details
    # OK, there are probably more elegant ways
    # of implementing the following code...meh
    refchandict=json.load(open(refchanfile))
    spw=[]
    for i in refchandict.keys():
        spw.append(int(i))
    
    nspw=len(spw)
    # json doesn't seem to load in the spw order properly
    # The user might not have entered spw's in order either
    # so perform sort just in case
    # note: no need to perform sort on the string versions
    spw.sort()
    # now get reference channels in corresponding sorted order
    refchan=[]
    for i in range(nspw):
        refchan.append(refchandict[str(spw[i])])
    
    # open MS and select relevant data
    ms.open(vis)
    ms.msselect({'field':str(field)})
    
    # get integration time
    scan_summary = ms.getscansummary()
    ms.close()
    scan_list = []
    for scan in scan_summary:
        if scan_summary[scan]['0']['FieldId'] == field:
            scan_list.append(int(scan))
    
    inttime=scan_summary[str(scan_list[0])]['0']['IntegrationTime']
    # get around potential floating point issues by rounding to nearest 1e-5 seconds
    if inttime != round(inttime,5):
        casalog.post('*** WARNING: It seems your integration time is specified to finer than 1e-5 seconds.','WARN')
        casalog.post('***          pieflag will assume this is a rounding error and carry on.','WARN')
    
    for i in range(len(scan_list)):
        if round(inttime,5) != round(scan_summary[str(scan_list[i])]['0']['IntegrationTime'],5):
            casalog.post('*** ERROR: Bummer, pieflag is not set up to handle '+\
                              'changing integration times throughout your MS.', 'ERROR')
            casalog.post('*** ERROR: Exiting pieflag.','ERROR')
            return
    
    # get number of baselines
    tb.open(vis+'/ANTENNA')
    atble=tb.getcol('NAME')
    tb.close()
    nant=atble.shape[0]
    nbaselines=nant*(nant-1)/2
    
    # channel to frequency (Hz) conversion
    tb.open(vis+'/SPECTRAL_WINDOW')
    vtble=tb.getcol('CHAN_FREQ')
    tb.close()
    # vtble format is vtble[channel][spw]
    # assume each spw has the same number of channels
    nchan=vtble.shape[0]
    
    # check that spw frequencies increase monotonically
    spwcheck=vtble[0,0]
    for s in range(1,len(vtble[0,:])):
        if vtble[0,s]<spwcheck:
            casalog.post("*** ERROR: Your spw's are not ordered with increasing frequency.",'ERROR')
            casalog.post('*** ERROR: Consider splitting your data and restarting pieflag. Exiting','ERROR')
            return
        
        spwcheck=vtble[0,s]
    
    # get number of polarizations, assume they don't change throughout observation
    # get details from the first user-selected spw within the first scan on target field
    # note: I won't assume that spw specifies data_desc_id in the main table, even
    #       though in most cases it probably does. Probably overkill given the lack
    #       of checks done elsewhere in this code...
    tb.open(vis+'/DATA_DESCRIPTION')
    temptb=tb.query('SPECTRAL_WINDOW_ID='+str(spw[0]))
    # while here, get the data_desc_id values that pair with spw number
    tempddid=tb.getcol('SPECTRAL_WINDOW_ID').tolist()
    ddid=[]
    for s in range(nspw):
        ddid.append(tempddid.index(spw[s]))
    
    tb.close()
    polid=temptb.getcell('POLARIZATION_ID')
    tb.open(vis+'/POLARIZATION')
    npol=tb.getcell('NUM_CORR',polid)
    poltype=tb.getcell('CORR_TYPE',polid)
    tb.close()
    
    if not (npol == 2 or npol == 4):
        casalog.post('*** ERROR: Your data contains '+str(npol)+' polarization products.','ERROR')
        casalog.post('*** ERROR: pieflag can only handle 2 (eg RR/LL) or 4 (eg RR/RL/LR/LL). Exiting.','ERROR')
        return
    
    # see stokes.h for details
    if poltype[0] == 5:
        # circular
        feedbasis = 1
    elif poltype[0] == 9:
        #linear
        feedbasis = 0
    else:
        casalog.post('*** ERROR: Your data uses an unsupported feed basis. Exiting','ERROR')
        return
    
    casalog.post('--> Some details about your data:')
    casalog.post('    data column to process = '+datacol)
    casalog.post('    integration time = '+str(inttime)+' sec')
    casalog.post('    number of baselines = '+str(nbaselines))
    casalog.post('    spectral windows to process = '+str(spw))
    casalog.post('    number of channels per spectral window = '+str(nchan))
    if feedbasis:
        casalog.post('    feed basis = circular')
    else:
        casalog.post('    feed basis = linear')
    
    casalog.post('    number of polarization products to process = '+str(npol))
    casalog.post('--> Statistics of pre-existing flags:')
    flag0 = np.zeros((nspw,2*npol+2))
    for i in range(nspw):
        casalog.filter('WARN')
        if useMPI:
            for k in range(nthreads):
                param = {'vis':vis+'/SUBMSS/'+subms[k],'field':field,\
                         'spw':spw[i],'npol':npol,'feedbasis':feedbasis}
                if k == 0:
                    pid = client.push_command_request(fcall1,False,None,param)
                else:
                    pid.append((client.push_command_request(fcall1,False,None,param))[0])
            
            presults = client.get_command_response(pid,True)
            for k in range(nthreads):
                flag0[i] += presults[k]['ret']
            
        else:
            flag0[i] = pieflag_getflagstats(vis,field,spw[i],npol,feedbasis)
        
        casalog.filter('INFO')
        RRs="{:.1f}".format(flag0[i][0]/flag0[i][1]*100.)
        LLs="{:.1f}".format(flag0[i][2]/flag0[i][3]*100.)
        TOTs="{:.1f}".format(flag0[i][4]/flag0[i][5]*100.)
        if npol == 2:
            if feedbasis:
                outstr='    flagged data in spw='+str(spw[i])+':  RR='+RRs+'%  LL='+LLs+'%  total='+TOTs+'%'
            else:
                outstr='    flagged data in spw='+str(spw[i])+':  XX='+RRs+'%  YY='+LLs+'%  total='+TOTs+'%'
        else:
            RLs="{:.1f}".format(flag0[i][6]/flag0[i][7]*100.)
            LRs="{:.1f}".format(flag0[i][8]/flag0[i][9]*100.)
            if feedbasis:
                outstr='    flagged data in spw='+str(spw[i])+':  RR='+RRs+'%  RL='+RLs+'%  LR='+LRs+'%  LL='+LLs+'%  total='+TOTs+'%'
            else:
                outstr='    flagged data in spw='+str(spw[i])+':  XX='+RRs+'%  XY='+RLs+'%  YX='+LRs+'%  YY='+LLs+'%  total='+TOTs+'%'
        
        casalog.post(outstr)
    
    # Check there are enough spectral windows to perform the fitting later on. If not, lower the order.
    if fitorder_RR_LL > nspw-1:
        if fitorder_RR_LL == 2:
            if feedbasis:
                casalog.post('*** WARNING: pieflag needs at least 3 spectral windows to fit for RR or LL spectral curvature.','WARN')
            else:
                casalog.post('*** WARNING: pieflag needs at least 3 spectral windows to fit for XX or YY spectral curvature.','WARN')
        else:
            if feedbasis:
                casalog.post('*** WARNING: pieflag needs at least 2 spectral windows to fit for RR or LL spectral index.','WARN')
            else:
                casalog.post('*** WARNING: pieflag needs at least 2 spectral windows to fit for XX or YY spectral index.','WARN')
        
        if nspw == 2:
            fitorder_RR_LL=1
        else:
            fitorder_RR_LL=0
        
        casalog.post('*** WARNING: fitorder_RR_LL has been reduced to '+str(int(fitorder_RR_LL))+ ' and','WARN')
        casalog.post('***          may be reduced further for some baselines if the','WARN')
        casalog.post('***          reference channel isn\'t available in all selected spw\'s.','WARN')
    
    if npol == 2:
        fitorder    = np.zeros(2)
        fitorder[0] = fitorder_RR_LL
        fitorder[1] = fitorder_RR_LL
    elif npol == 4:
        if fitorder_RL_LR > nspw-1:
            if fitorder_RL_LR == 2:
                casalog.post('*** WARNING: pieflag needs at least 3 spectral windows to fit for RL or LR spectral curvature.','WARN')
            else:
                casalog.post('*** WARNING: pieflag needs at least 2 spectral windows to fit for RL or LR spectral index.','WARN')
            
            if nspw == 2:
                fitorder_RL_LR=1
            else:
                fitorder_RL_LR=0
            
            casalog.post('*** WARNING: fitorder_RL_LR has been reduced to '+str(int(fitorder_RL_LR))+' and','WARN')
            casalog.post('***          may be reduced further for some baselines if the','WARN')
            casalog.post('***          reference channel isn\'t available in all selected spw\'s.','WARN')
        
        fitorder    = np.zeros(4)
        fitorder[0] = fitorder_RR_LL
        fitorder[1] = fitorder_RL_LR
        fitorder[2] = fitorder_RL_LR
        fitorder[3] = fitorder_RR_LL
    
    if scalethresh:
        # read in SEFD data and interpolate to get values at our channel frequencies
        casalog.post('--> Reading in SEFD and interpolating at channel frequencies...')
        sefdRAW=np.loadtxt(SEFDfile)
        sefd=np.zeros((nspw,nchan))
        if not np.all(np.diff(sefdRAW[:,0]) >= 0):
            casalog.post('*** ERROR: Your SEFD file must be in order of increasing frequency.','ERROR')
            casalog.post('*** ERROR: Exiting pieflag.','ERROR')
            return
        
        for i in range(nspw):
            if (vtble[:,spw[i]].min() < sefdRAW[:,0].min()) or (vtble[:,spw[i]].max() > sefdRAW[:,0].max()):
                casalog.post('*** ERROR: pieflag cannot extrapolate your SEFD.','ERROR')
                casalog.post('*** ERROR: Provide new SEFD covering your entire frequency range.','ERROR')
                casalog.post('*** ERROR: Exiting pieflag.','ERROR')
                return
        
        sefdINTERP = interp1d(sefdRAW[:,0],sefdRAW[:,1])
        for i in range(nspw):
            sefdREFCHAN = sefdINTERP(vtble[refchan[i]][spw[i]])
            for j in range(nchan):
                # values in each spectral window will be relative to the reference channel value
                sefd[i][j] = sefdINTERP(vtble[j][spw[i]]) / sefdREFCHAN
        
        if plotSEFD:
            # clunky, but works, meh...
            sefdPLOT=np.zeros((nspw*nchan,3))
            k=0
            for i in range(nspw):
                sefdREFCHAN = sefdINTERP(vtble[refchan[i]][spw[i]])
                for j in range(nchan):
                    sefdPLOT[k][0] = vtble[j][spw[i]]/1.0e9
                    sefdPLOT[k][1] = sefd[i][j] * sefdREFCHAN
                    sefdPLOT[k][2] = sefd[i][j]
                    k += 1
            
            f, (ax1, ax2) = plt.subplots(2,sharex=True)
            ax1.plot(sefdRAW[:,0]/1.0e9,sefdRAW[:,1],'b-',sefdPLOT[:,0],sefdPLOT[:,1],'r.',markersize=10)
            ax2.plot([sefdRAW[0,0]/1.0e9,sefdRAW[len(sefdRAW[:,0])-1,0]/1.0e9],[1.,1.],'c-',sefdPLOT[:,0],sefdPLOT[:,2],'r.',markersize=10)
            f.subplots_adjust(hspace=0)
            plt.setp([a.get_xticklabels() for a in f.axes[:-1]], visible=False)
            ax1.set_title('relative sensitivity assumed across your band,\nnormalized to the reference channel in each spw')
            ax1.legend(['raw input','interpolated'])
            ax1.set_ylabel('SEFD (arbitrary units)')
            ax2.set_xlabel('frequency (GHz)')
            ax2.set_ylabel('SEFD (normalized units per spw)')
    else:
        sefd=np.ones((nspw,nchan))
    
    if not staticflag:
        madmax = 0
        binsamples = 0
    
    if not dynamicflag:
        chunktime = 0
        stdmax = 0
        maxoffset = 0
    
    if not extendflag:
        boxtime = 0
        boxthresh = 0
    
    # forcibly remove all lock files
    #os.system('find '+vis+' -name "*lock" -print | xargs rm')
    
    if useMPI:
        casalog.post('--> pieflag will now flag your data using '+str(nthreads)+' parallel threads.')
        casalog.filter('WARN')
        for k in range(nthreads):
            param = {'vis':vis+'/SUBMSS/'+subms[k],'datacol':datacol,'nthreads':nthreads,'field':field,
                     'vtbleLIST':vtble.tolist(),'inttime':inttime,'nant':nant,
                     'ddid':ddid,'spw':spw,'refchan':refchan,'nchan':nchan,'npol':npol,'feedbasis':feedbasis,
                     'fitorderLIST':fitorder.tolist(),'sefdLIST':sefd.tolist(),
                     'staticflag':staticflag,'madmax':madmax,'binsamples':binsamples,
                     'dynamicflag':dynamicflag,'chunktime':chunktime,'stdmax':stdmax,'maxoffset':maxoffset,
                     'extendflag':extendflag,'boxtime':boxtime,'boxthresh':boxthresh}
            if k == 0:
                pid = client.push_command_request(fcall2,False,None,param)
            else:
                pid.append((client.push_command_request(fcall2,False,None,param))[0])
        
        presults = client.get_command_response(pid,True)
        casalog.filter('INFO')
    else:
        casalog.post('--> pieflag will now flag your data in serial mode.')
        pieflag_flag(vis,datacol,1,field,
                     vtble.tolist(),inttime,nant,
                     ddid,spw,refchan,nchan,npol,feedbasis,
                     fitorder.tolist(),sefd.tolist(),
                     staticflag,madmax,binsamples,
                     dynamicflag,chunktime,stdmax,maxoffset,
                     extendflag,boxtime,boxthresh)
    
    # show updated flagging statistics
    casalog.post('--> Statistics of final flags (including pre-existing):')
    flag1 = np.zeros((nspw,2*npol+2))
    for i in range(nspw):
        casalog.filter('WARN')
        if useMPI:
            for k in range(nthreads):
                param = {'vis':vis+'/SUBMSS/'+subms[k],'field':field,\
                         'spw':spw[i],'npol':npol,'feedbasis':feedbasis}
                if k == 0:
                    pid = client.push_command_request(fcall1,False,None,param)
                else:
                    pid.append((client.push_command_request(fcall1,False,None,param))[0])
            
            presults = client.get_command_response(pid,True)
            for k in range(nthreads):
                flag1[i] += presults[k]['ret']
            
        else:
            flag1[i] = pieflag_getflagstats(vis,field,spw[i],npol,feedbasis)
        
        casalog.filter('INFO')
        RRs="{:.1f}".format(flag1[i][0]/flag1[i][1]*100.)
        LLs="{:.1f}".format(flag1[i][2]/flag1[i][3]*100.)
        TOTs="{:.1f}".format(flag1[i][4]/flag1[i][5]*100.)
        if npol == 2:
            if feedbasis:
                outstr='    flagged data in spw='+str(spw[i])+':  RR='+RRs+'%  LL='+LLs+'%  total='+TOTs+'%'
            else:
                outstr='    flagged data in spw='+str(spw[i])+':  XX='+RRs+'%  YY='+LLs+'%  total='+TOTs+'%'
        else:
            RLs="{:.1f}".format(flag1[i][6]/flag1[i][7]*100.)
            LRs="{:.1f}".format(flag1[i][8]/flag1[i][9]*100.)
            if feedbasis:
                outstr='    flagged data in spw='+str(spw[i])+':  RR='+RRs+'%  RL='+RLs+'%  LR='+LRs+'%  LL='+LLs+'%  total='+TOTs+'%'
            else:
                outstr='    flagged data in spw='+str(spw[i])+':  XX='+RRs+'%  XY='+RLs+'%  YX='+LRs+'%  YY='+LLs+'%  total='+TOTs+'%'
        
        casalog.post(outstr)
    
    casalog.post('--> Statistics of pieflag flags (excluding pre-existing):')
    for i in range(nspw):
        RRs="{:.1f}".format((flag1[i][0]-flag0[i][0])/flag0[i][1]*100.)
        LLs="{:.1f}".format((flag1[i][2]-flag0[i][2])/flag0[i][3]*100.)
        TOTs="{:.1f}".format((flag1[i][4]-flag0[i][4])/flag0[i][5]*100.)
        if npol == 2:
            if feedbasis:
                outstr='    data flagged in spw='+str(spw[i])+':  RR='+RRs+'%  LL='+LLs+'%  total='+TOTs+'%'
            else:
                outstr='    data flagged in spw='+str(spw[i])+':  XX='+RRs+'%  YY='+LLs+'%  total='+TOTs+'%'
        else:
            RLs="{:.1f}".format((flag1[i][6]-flag0[i][6])/flag0[i][7]*100.)
            LRs="{:.1f}".format((flag1[i][8]-flag0[i][8])/flag0[i][9]*100.)
            if feedbasis:
                outstr='    data flagged in spw='+str(spw[i])+':  RR='+RRs+'%  RL='+RLs+'%  LR='+LRs+'%  LL='+LLs+'%  total='+TOTs+'%'
            else:
                outstr='    data flagged in spw='+str(spw[i])+':  XX='+RRs+'%  XY='+RLs+'%  YX='+LRs+'%  YY='+LLs+'%  total='+TOTs+'%'
        
        casalog.post(outstr)
    
    # forcibly remove all lock files
    #os.system('find '+vis+' -name "*lock" -print | xargs rm')
    
    if useMPI:
        #client.set_log_level('INFO')
        client.push_command_request("casalog.filter('INFO')",True,server_list)
    
    t=time.time()-startTime
    casalog.post('--> pieflag run time:  '+str(int(t//3600))+' hours  '+\
                 str(int(t%3600//60))+' minutes  '+str(int(t%60))+' seconds')