Example #1
0
    def test_nomms(self):
        '''Partition: Create a normal MS with createmms=False'''
        partition(vis=self.msfile, outputvis=self.mmsfile, createmms=False)

        self.assertTrue(os.path.exists(self.mmsfile),
                        'MMS was not created for this test')

        # Sort the output MSs so that they can be compared
        myms = mstool()

        myms.open(self.msfile)
        myms.sort('ms_sorted.ms', [
            'OBSERVATION_ID', 'ARRAY_ID', 'SCAN_NUMBER', 'FIELD_ID',
            'DATA_DESC_ID', 'ANTENNA1', 'ANTENNA2', 'TIME'
        ])
        myms.done()

        myms.open(self.mmsfile)
        myms.sort('mms_sorted.ms', [
            'OBSERVATION_ID', 'ARRAY_ID', 'SCAN_NUMBER', 'FIELD_ID',
            'DATA_DESC_ID', 'ANTENNA1', 'ANTENNA2', 'TIME'
        ])
        myms.done()

        # Compare both tables. Ignore the DATA column and compare it in next line
        self.assertTrue(
            th.compTables(
                'ms_sorted.ms', 'mms_sorted.ms',
                ['FLAG_CATEGORY', 'FLAG', 'WEIGHT_SPECTRUM', 'DATA']))

        # Compare the DATA column
        self.assertTrue(
            th.compVarColTables('ms_sorted.ms', 'mms_sorted.ms', 'DATA'))
Example #2
0
    def test_scan_selection(self):
        """Partition: create an MMS using scan selection"""
        partition(
            vis=self.msfile,
            outputvis=self.mmsfile,
            separationaxis="scan",
            scan="1,2,3,11",
            flagbackup=False,
            disableparallel=True,
        )

        self.assertTrue(os.path.exists(self.mmsfile), "MMS was not created for this test")

        # Take the dictionary and compare with original MS
        thisdict = listpartition(vis=self.mmsfile, createdict=True)

        # Compare nrows of all scans in selection
        slist = ph.getMMSScans(thisdict)
        self.assertEqual(slist.__len__(), 4)

        for s in slist:
            mmsN = ph.getMMSScanNrows(thisdict, s)
            msN = ph.getScanNrows(self.msfile, s)
            self.assertEqual(mmsN, msN, "Nrows in scan=%s differs: mms_nrows=%s <--> ms_nrows=%s" % (s, mmsN, msN))

        # Compare spw IDs
        for s in slist:
            mms_spw = ph.getSpwIds(self.mmsfile, s)
            ms_spw = ph.getSpwIds(self.msfile, s)
            self.assertEqual(
                mms_spw, ms_spw, "list of spws in scan=%s differs: " "mms_spw=%s <--> ms_spw=%s" % (s, mmsN, msN)
            )
Example #3
0
    def runPartition(self, ms, mmsdir, createlink, axis, subms):
        '''Run partition with default values to create an MMS.
           ms         --> full pathname of the MS
           mmsdir     --> directory to save the MMS to
           createlink --> when True, it will create a symbolic link to the
                         just created MMS in the same directory with extension .ms  
           axis      --> separationaxis to use (spw, scan, auto)
           subms  --> number of subMss to create

        '''
        from tasks import partition

        if not os.path.lexists(ms):
            return False

        # Create MMS name
        bname = os.path.basename(ms)
        if bname.endswith('.ms'):
            mmsname = bname.replace('.ms', '.mms')
        else:
            mmsname = bname + '.mms'

        mms = os.path.join(self.mmsdir, mmsname)
        if os.path.lexists(mms):
            casalog.post('Output MMS already exist -->' + mms, 'ERROR')
            return False

        # Check for remainings of corrupted mms
        corrupted = mms.replace('.mms', '.data')
        if os.path.exists(corrupted):
            casalog.post('Cleaning up left overs', 'WARN')
            shutil.rmtree(corrupted)

        # Run partition
        default('partition')
        partition(vis=ms,
                  outputvis=mms,
                  createmms=True,
                  datacolumn='all',
                  flagbackup=False,
                  separationaxis=axis,
                  numsubms=subms)
        casalog.origin('convertToMMS')

        # Check if MMS was created
        if not os.path.exists(mms):
            casalog.post('Cannot create MMS ->' + mms, 'ERROR')
            return False

        # If requested, create a link to this MMS
        if createlink:
            here = os.getcwd()
            os.chdir(mmsdir)
            mmsname = os.path.basename(mms)
            lms = mmsname.replace('.mms', '.ms')
            casalog.post('Creating symbolic link to MMS')
            os.symlink(mmsname, lms)
            os.chdir(here)

        return True
Example #4
0
    def test_flagsrestore(self):
        """Partition: check that we can restore the flags"""
        # Delete any flagversions
        if os.path.exists(self.msfile + ".flagversions"):
            shutil.rmtree(self.msfile + ".flagversions")

        # Unflag the MS
        flagdata(vis=self.msfile, mode="unflag", flagbackup=False)

        # Run partition and create the .flagversions
        partition(vis=self.msfile, outputvis=self.mmsfile, createmms=True, disableparallel=True, datacolumn="data")
        self.assertTrue(os.path.exists(self.mmsfile + ".flagversions"))

        # Flag spw=9 and then spw=7 in the MMS
        flagdata(vis=self.mmsfile, mode="manual", spw="9", flagbackup=True)
        flagdata(vis=self.mmsfile, mode="manual", spw="7", flagbackup=True)

        # There should be flags in spw=7 and 9 in MMS
        res = flagdata(vis=self.mmsfile, mode="summary")
        self.assertEqual(res["flagged"], 549888)

        # Restore the original flags (there should be none)
        flagmanager(vis=self.mmsfile, mode="restore", versionname="partition_1")
        res = flagdata(vis=self.mmsfile, mode="summary")
        self.assertEqual(res["flagged"], 0)
Example #5
0
    def test_all_columns(self):
        '''Partition: datacolumn=all'''
        partition(vis=self.msfile,
                  outputvis=self.mmsfile,
                  datacolumn='all',
                  flagbackup=False,
                  disableparallel=True)

        self.assertTrue(os.path.exists(self.mmsfile),
                        'MMS was not created for this test')

        # Take the dictionary and compare with original MS
        thisdict = listpartition(vis=self.mmsfile, createdict=True)

        # Compare nrows of all scans in selection
        slist = ph.getMMSScans(thisdict)
        self.assertEqual(slist.__len__(), 2)
        for s in slist:
            mmsN = ph.getMMSScanNrows(thisdict, s)
            msN = ph.getScanNrows(self.msfile, s)
            self.assertEqual(
                mmsN, msN,
                'Nrows in scan=%s differs: mms_nrows=%s <--> ms_nrows=%s' %
                (s, mmsN, msN))

        # Compare spw IDs
        for s in slist:
            mms_spw = ph.getSpwIds(self.mmsfile, s)
            ms_spw = ph.getSpwIds(self.msfile, s)
            self.assertEqual(mms_spw, ms_spw, 'list of spws in scan=%s differs: '\
                             'mms_spw=%s <--> ms_spw=%s' %(s, mms_spw, ms_spw))
Example #6
0
    def test_default_sequential(self):
        '''Partition: create an MMS with default values in sequential'''
        partition(vis=self.msfile,
                  outputvis=self.mmsfile,
                  disableparallel=True)

        self.assertTrue(os.path.exists(self.mmsfile),
                        'MMS was not created for this test')

        # Take the dictionary and compare with original MS
        thisdict = listpartition(vis=self.mmsfile, createdict=True)

        # Get scans of MMS
        slist = ph.getMMSScans(thisdict)

        for s in slist:
            mmsN = ph.getMMSScanNrows(thisdict, s)
            msN = ph.getScanNrows(self.msfile, s)
            self.assertEqual(
                mmsN, msN,
                'Nrows in scan=%s differs: mms_nrows=%s <--> ms_nrows=%s' %
                (s, mmsN, msN))

        # Compare spw IDs of MS and MMS
        for s in slist:
            mms_spw = ph.getSpwIds(self.mmsfile, s)
            ms_spw = ph.getSpwIds(self.msfile, s)
            self.assertEqual(mms_spw, ms_spw, 'list of spws in scan=%s differs: '\
                             'mms_spw=%s <--> ms_spw=%s' %(s, mmsN, msN))

        # The separation axis should be written to the output MMS
        sepaxis = ph.axisType(self.mmsfile)
        self.assertEqual(sepaxis, 'scan,spw',
                         'Partition did not write AxisType correctly in MMS')
Example #7
0
    def createMMS(self,
                  msfile,
                  axis='auto',
                  scans='',
                  spws='',
                  fields='',
                  numms=64):
        '''Create MMSs for tests with input MMS'''
        prefix = msfile[:-len('.ms')]

        # Create an MMS for the tests
        self.testmms = "cvel_input_test.mms"
        default(partition)

        if os.path.exists(self.testmms):
            os.system("rm -rf " + self.testmms)
            os.system("rm -rf " + self.testmms + '.flagversions')

        print "................. Creating test MMS .................."
        partition(vis=msfile,
                  outputvis=self.testmms,
                  flagbackup=False,
                  separationaxis=axis,
                  scan=scans,
                  spw=spws,
                  field=fields,
                  numsubms=numms)
Example #8
0
    def test_spw_selection(self):
        '''Partition: create an MMS separated by spws with spw=2,4 selection'''
        partition(vis=self.msfile, outputvis=self.mmsfile, separationaxis='spw', spw='2,4',
                  flagbackup=False, disableparallel=True)
        
        self.assertTrue(os.path.exists(self.mmsfile), 'MMS was not created for this test')
        
        # Take the dictionary and compare with original MS
        thisdict = listpartition(vis=self.mmsfile, createdict=True)
        
        # Dictionary with selection to compare with original MS
        mysel = {'spw':'2,4'}
        
        # Compare nrows of all scans in selection
        slist = ph.getMMSScans(thisdict)
        for s in slist:
            mmsN = ph.getMMSScanNrows(thisdict, s)
            msN = ph.getScanNrows(self.msfile, s, selection=mysel)
            self.assertEqual(mmsN, msN, 'Nrows in scan=%s differs: mms_nrows=%s <--> ms_nrows=%s'
                             %(s, mmsN, msN))
 
        # spwids are re-indexed. The expected IDs are:
        # ms_spw = 2 --> mms_spw = 0
        # ms_spw = 4 --> mms_spw = 1
        # Check that MMS spw IDs have been re-indexed properly
        indexed_ids = range(2)
        for s in slist:
            mms_spw = ph.getSpwIds(self.mmsfile, s)
            self.assertEqual(mms_spw, indexed_ids, 'spw IDs were not properly re-indexed')
Example #9
0
    def test_spw_separation(self):
        '''Partition: create an MMS separated by spws with observation selection'''
        # NOTE: ms.getscansummary() used in ph.getScanNrows does not honour several observation
        #       IDs, therefore I need to selection by obs id in partition
        partition(vis=self.msfile, outputvis=self.mmsfile, separationaxis='spw', observation='2',
                  flagbackup=False, disableparallel=True)
        
        self.assertTrue(os.path.exists(self.mmsfile), 'MMS was not created for this test')
        
        # Take the dictionary and compare with original MS
        thisdict = listpartition(vis=self.mmsfile, createdict=True)

        # Dictionary with selection to compare with original MS
        mysel = {'observation':'2'}
        
        # Compare nrows of all scans in MMS and MS
        slist = ph.getMMSScans(thisdict)
        for s in slist:
            mmsN = ph.getMMSScanNrows(thisdict, s)
            msN = ph.getScanNrows(self.msfile, s, selection=mysel)
            self.assertEqual(mmsN, msN, 'Nrows in scan=%s differs: mms_nrows=%s <--> ms_nrows=%s'
                             %(s, mmsN, msN))
 
        # spwids are re-indexed. The expected IDs are:
        # ms_spw = 2 --> mms_spw = 0
        # ms_spw = 3 --> mms_spw = 1, etc.
        # Check that MMS spw IDs have been re-indexed properly
        indexed_ids = range(4)
        for s in slist:
            mms_spw = ph.getSpwIds(self.mmsfile, s)
            self.assertEqual(mms_spw, indexed_ids, 'spw IDs were not properly re-indexed')
Example #10
0
    def test_sepaxis(self):
        '''Partition: separationaxis=auto'''        
        partition(vis=self.msfile, outputvis=self.mmsfile, spw='0~11',separationaxis='auto',
                  flagbackup=False, disableparallel=True, datacolumn='data')
#        partition(vis=self.msfile, outputvis=self.mmsfile,separationaxis='auto')
        
        self.assertTrue(os.path.exists(self.mmsfile), 'MMS was not created for this test')

        # Dictionary with selection to compare with original MS
        mysel = {'spw':'0~11'}
        
        # Take the dictionary and compare with original MS
        thisdict = listpartition(vis=self.mmsfile, createdict=True)
        
        # Compare nrows of all scans in selection
        slist = ph.getMMSScans(thisdict)
        self.assertEqual(slist.__len__(), 2)
        for s in slist:
            mmsN = ph.getMMSScanNrows(thisdict, s)
            msN = ph.getScanNrows(self.msfile, s, selection=mysel)
#            msN = ph.getScanNrows(self.msfile, s)
            self.assertEqual(mmsN, msN, 'Nrows in scan=%s differs: mms_nrows=%s <--> ms_nrows=%s'
                             %(s, mmsN, msN))

        # Compare spw IDs
        for s in slist:
            mms_spw = ph.getSpwIds(self.mmsfile, s)
            ms_spw = ph.getSpwIds(self.msfile, s, selection=mysel)
#            ms_spw = ph.getSpwIds(self.msfile, s)
            self.assertEqual(mms_spw, ms_spw, 'list of spws in scan=%s differs: '\
                             'mms_spw=%s <--> ms_spw=%s' %(s, mms_spw, ms_spw))

        # The separation axis should be written to the output MMS
        sepaxis = ph.axisType(self.mmsfile)
        self.assertEqual(sepaxis, 'scan,spw', 'Partition did not write AxisType correctly in MMS')
Example #11
0
    def test_default_sequential(self):
        '''Partition: create an MMS with default values in sequential'''
        partition(vis=self.msfile, outputvis=self.mmsfile, disableparallel=True)
        
        self.assertTrue(os.path.exists(self.mmsfile), 'MMS was not created for this test')
        
        # Take the dictionary and compare with original MS
        thisdict = listpartition(vis=self.mmsfile, createdict=True)
        
        # Get scans of MMS
        slist = ph.getMMSScans(thisdict)
         
        for s in slist:
            mmsN = ph.getMMSScanNrows(thisdict, s)
            msN = ph.getScanNrows(self.msfile, s)
            self.assertEqual(mmsN, msN, 'Nrows in scan=%s differs: mms_nrows=%s <--> ms_nrows=%s'
                             %(s, mmsN, msN))
 
        # Compare spw IDs of MS and MMS
        for s in slist:
            mms_spw = ph.getSpwIds(self.mmsfile, s)
            ms_spw = ph.getSpwIds(self.msfile, s)
            self.assertEqual(mms_spw, ms_spw, 'list of spws in scan=%s differs: '\
                             'mms_spw=%s <--> ms_spw=%s' %(s, mmsN, msN))
            
        # The separation axis should be written to the output MMS
        sepaxis = ph.axisType(self.mmsfile)
        self.assertEqual(sepaxis, 'scan,spw', 'Partition did not write AxisType correctly in MMS')
Example #12
0
    def test_flagversions(self):
        '''Partition: check that the .flagversions is created'''

        # Run partition and create the .flagversions
        partition(vis=self.msfile,
                  outputvis=self.mmsfile,
                  createmms=True,
                  disableparallel=True,
                  datacolumn='data')
        self.assertTrue(os.path.exists(self.mmsfile + '.flagversions'))

        # Check that the number of backups in MMS is correct
        aflocal = aftool()
        aflocal.open(self.mmsfile)
        nv = aflocal.getflagversionlist()
        aflocal.done()
        self.assertEqual(len(nv), 3)

        # Run flagdata on MMS to check if it works well.
        flagdata(vis=self.mmsfile, mode='unflag', flagbackup=True)

        # Check that the number of backups in MMS is correct
        aflocal = aftool()
        aflocal.open(self.mmsfile)
        nvref = aflocal.getflagversionlist()
        aflocal.done()
        self.assertEqual(len(nvref), 4)
Example #13
0
    def test_scan_spw(self):
        '''Partition: separationaxis=scan with spw selection'''
        partition(vis=self.msfile, outputvis=self.mmsfile, separationaxis='scan',
                  spw='1~4,10,11', flagbackup=False, datacolumn='data')
        
        self.assertTrue(os.path.exists(self.mmsfile), 'MMS was not created for this test')
        self.assertTrue(os.path.exists(self.msfile),'Make sure the input MS is not deleted inside the task')
        
        # Take the dictionary and compare with original MS
        thisdict = listpartition(vis=self.mmsfile, createdict=True)

        # Dictionary with selection to compare with original MS
        mysel = {'spw':'1~4,10,11'}
        
        # Compare nrows of all scans in selection
        slist = ph.getMMSScans(thisdict)
        for s in slist:
            mmsN = ph.getMMSScanNrows(thisdict, s)
            msN = ph.getScanNrows(self.msfile, s, selection=mysel)
            self.assertEqual(mmsN, msN, 'Nrows in scan=%s differs: mms_nrows=%s <--> ms_nrows=%s'
                             %(s, mmsN, msN))

        # The comparison should be
        # ms_spw = 1 --> mms_spw = 0
        # ms_spw = 2 --> mms_spw = 1, etc.
        # Check that MMS spw IDs have been re-indexed properly
        indexed_ids = range(6)
        for s in slist:
            mms_spw = ph.getSpwIds(self.mmsfile, s)
            self.assertEqual(mms_spw, indexed_ids, 'spw IDs were not properly re-indexed')
            
        # The separation axis should be written to the output MMS
        sepaxis = ph.axisType(self.mmsfile)
        self.assertEqual(sepaxis, 'scan', 'Partition did not write AxisType correctly in MMS')
Example #14
0
    def test_all_columns(self):
        '''Partition: datacolumn=all'''
        partition(vis=self.msfile, outputvis=self.mmsfile, datacolumn='all',
                  flagbackup=False, disableparallel=True)
        
        self.assertTrue(os.path.exists(self.mmsfile), 'MMS was not created for this test')
        
        # Take the dictionary and compare with original MS
        thisdict = listpartition(vis=self.mmsfile, createdict=True)
        
        # Compare nrows of all scans in selection
        slist = ph.getMMSScans(thisdict)
        self.assertEqual(slist.__len__(), 2)
        for s in slist:
            mmsN = ph.getMMSScanNrows(thisdict, s)
            msN = ph.getScanNrows(self.msfile, s)
            self.assertEqual(mmsN, msN, 'Nrows in scan=%s differs: mms_nrows=%s <--> ms_nrows=%s'
                             %(s, mmsN, msN))

        # Compare spw IDs
        for s in slist:
            mms_spw = ph.getSpwIds(self.mmsfile, s)
            ms_spw = ph.getSpwIds(self.msfile, s)
            self.assertEqual(mms_spw, ms_spw, 'list of spws in scan=%s differs: '\
                             'mms_spw=%s <--> ms_spw=%s' %(s, mms_spw, ms_spw))
Example #15
0
 def test_flagsrestore(self):
     '''Partition: check that we can restore the flags'''
     # Delete any flagversions
     if os.path.exists(self.msfile+'.flagversions'):
         shutil.rmtree(self.msfile+'.flagversions')
         
     # Unflag the MS
     flagdata(vis=self.msfile, mode='unflag', flagbackup=False)
     
     # Run partition and create the .flagversions
     partition(vis=self.msfile, outputvis=self.mmsfile, createmms=True,
               disableparallel=True, datacolumn='data')
     self.assertTrue(os.path.exists(self.mmsfile+'.flagversions'))
     
     # Flag spw=9 and then spw=7 in the MMS
     flagdata(vis=self.mmsfile, mode='manual', spw='9', flagbackup=True)
     flagdata(vis=self.mmsfile, mode='manual', spw='7', flagbackup=True)
     
     # There should be flags in spw=7 and 9 in MMS
     res = flagdata(vis=self.mmsfile, mode='summary')
     self.assertEqual(res['flagged'],549888)
     
     # Restore the original flags (there should be none)
     flagmanager(vis=self.mmsfile, mode='restore', versionname='partition_1')
     res = flagdata(vis=self.mmsfile, mode='summary')
     self.assertEqual(res['flagged'],0)
Example #16
0
 def test_merge_subtables_after_partiton_alma(self):
     '''mstransform: Check that sub-tables are properly merged after partiton'''
     
     self.outputms = "parted-alma.ms"
     
     partition(self.vis, outputvis=self.outputms,separationaxis='spw',numsubms=4,flagbackup=False)
     
     subtable = "/FEED"
     sort_order = "SPECTRAL_WINDOW_ID, ANTENNA_ID, FEED_ID, TIME"
     res = th.compareSubTables(self.outputms+subtable,self.vis+subtable,sort_order)
     self.assertTrue(res[0],"Error comparing " + res[1] + " column from " + subtable + " sub-table")
     
     subtable = "/CALDEVICE"
     sort_order = "SPECTRAL_WINDOW_ID, ANTENNA_ID, FEED_ID, TIME"
     excluded_cols = ['CAL_LOAD_NAMES','NOISE_CAL','CAL_EFF','TEMPERATURE_LOAD']
     res = th.compareSubTables(self.outputms+subtable,self.vis+subtable,sort_order,excluded_cols)
     self.assertTrue(res[0],"Error comparing " + res[1] + " column from " + subtable + " sub-table")       
     
     subtable = "/SYSCAL"
     sort_order = "SPECTRAL_WINDOW_ID, ANTENNA_ID, FEED_ID, TIME"
     excluded_cols = ['TCAL_SPECTRUM','TRX_SPECTRUM','TSKY_SPECTRUM','TSYS_SPECTRUM','TANT_SPECTRUM','TANT_TSYS_SPECTRUM']
     res = th.compareSubTables(self.outputms+subtable,self.vis+subtable,sort_order,excluded_cols)
     self.assertTrue(res[0],"Error comparing " + res[1] + " column from " + subtable + " sub-table")   
     
     subtable = "/SOURCE"
     sort_order = "SPECTRAL_WINDOW_ID, TIME, SOURCE_ID"
     excluded_cols = ['POSITION','TRANSITION','REST_FREQUENCY','SYSVEL']        
     res = th.compareSubTables(self.outputms+subtable,self.vis+subtable,sort_order,excluded_cols)
     self.assertTrue(res[0],"Error comparing " + res[1] + " column from " + subtable + " sub-table")         
     
     subtable = "/SPECTRAL_WINDOW"
     excluded_cols = ['ASSOC_SPW_ID','ASSOC_NATURE']
     res = th.compareSubTables(self.outputms+subtable,self.vis+subtable,None,excluded_cols)
     self.assertTrue(res[0],"Error comparing " + res[1] + " column from " + subtable + " sub-table")           
Example #17
0
    def test_flagsrestore(self):
        '''Partition: check that we can restore the flags'''
        # Delete any flagversions
        if os.path.exists(self.msfile + '.flagversions'):
            shutil.rmtree(self.msfile + '.flagversions')

        # Unflag the MS
        flagdata(vis=self.msfile, mode='unflag', flagbackup=False)

        # Run partition and create the .flagversions
        partition(vis=self.msfile,
                  outputvis=self.mmsfile,
                  createmms=True,
                  disableparallel=True,
                  datacolumn='data')
        self.assertTrue(os.path.exists(self.mmsfile + '.flagversions'))

        # Flag spw=9 and then spw=7 in the MMS
        flagdata(vis=self.mmsfile, mode='manual', spw='9', flagbackup=True)
        flagdata(vis=self.mmsfile, mode='manual', spw='7', flagbackup=True)

        # There should be flags in spw=7 and 9 in MMS
        res = flagdata(vis=self.mmsfile, mode='summary')
        self.assertEqual(res['flagged'], 549888)

        # Restore the original flags (there should be none)
        flagmanager(vis=self.mmsfile,
                    mode='restore',
                    versionname='partition_1')
        res = flagdata(vis=self.mmsfile, mode='summary')
        self.assertEqual(res['flagged'], 0)
Example #18
0
    def test_merge_subtables_after_partiton_evla(self):
        """mstransform: Check that sub-tables are properly merged after partiton"""

        self.outputms = "parted-evla.ms"

        partition(self.vis, outputvis=self.outputms, separationaxis="spw", numsubms=4, flagbackup=False)

        subtable = "/FEED"
        sort_order = "SPECTRAL_WINDOW_ID, ANTENNA_ID, FEED_ID, TIME"
        res = th.compareSubTables(self.outputms + subtable, self.vis + subtable, sort_order)
        self.assertTrue(res[0], "Error comparing " + res[1] + " column from " + subtable + " sub-table")

        subtable = "/CALDEVICE"
        sort_order = "SPECTRAL_WINDOW_ID, ANTENNA_ID, FEED_ID, TIME"
        excluded_cols = ["CAL_LOAD_NAMES", "NOISE_CAL", "CAL_EFF", "TEMPERATURE_LOAD"]
        res = th.compareSubTables(self.outputms + subtable, self.vis + subtable, sort_order, excluded_cols)
        self.assertTrue(res[0], "Error comparing " + res[1] + " column from " + subtable + " sub-table")

        subtable = "/SYSPOWER"
        sort_order = "SPECTRAL_WINDOW_ID, ANTENNA_ID, FEED_ID, TIME"
        excluded_cols = ["SWITCHED_DIFF", "SWITCHED_SUM", "REQUANTIZER_GAIN", "TANT_SPECTRUM"]
        res = th.compareSubTables(self.outputms + subtable, self.vis + subtable, sort_order, excluded_cols)
        self.assertTrue(res[0], "Error comparing " + res[1] + " column from " + subtable + " sub-table")

        subtable = "/SOURCE"
        sort_order = "SPECTRAL_WINDOW_ID, TIME, SOURCE_ID"
        excluded_cols = ["POSITION", "TRANSITION", "REST_FREQUENCY", "SYSVEL"]
        res = th.compareSubTables(self.outputms + subtable, self.vis + subtable, sort_order, excluded_cols)
        self.assertTrue(res[0], "Error comparing " + res[1] + " column from " + subtable + " sub-table")

        subtable = "/SPECTRAL_WINDOW"
        excluded_cols = ["ASSOC_SPW_ID", "ASSOC_NATURE"]
        res = th.compareSubTables(self.outputms + subtable, self.vis + subtable, None, excluded_cols)
        self.assertTrue(res[0], "Error comparing " + res[1] + " column from " + subtable + " sub-table")
Example #19
0
    def test_nomms(self):
        """Partition: Create a normal MS with createmms=False"""
        partition(vis=self.msfile, outputvis=self.mmsfile, createmms=False)

        self.assertTrue(os.path.exists(self.mmsfile), "MMS was not created for this test")

        # Sort the output MSs so that they can be compared
        myms = mstool()

        myms.open(self.msfile)
        myms.sort(
            "ms_sorted.ms",
            ["OBSERVATION_ID", "ARRAY_ID", "SCAN_NUMBER", "FIELD_ID", "DATA_DESC_ID", "ANTENNA1", "ANTENNA2", "TIME"],
        )
        myms.done()

        myms.open(self.mmsfile)
        myms.sort(
            "mms_sorted.ms",
            ["OBSERVATION_ID", "ARRAY_ID", "SCAN_NUMBER", "FIELD_ID", "DATA_DESC_ID", "ANTENNA1", "ANTENNA2", "TIME"],
        )
        myms.done()

        # Compare both tables. Ignore the DATA column and compare it in next line
        self.assertTrue(
            th.compTables("ms_sorted.ms", "mms_sorted.ms", ["FLAG_CATEGORY", "FLAG", "WEIGHT_SPECTRUM", "DATA"])
        )

        # Compare the DATA column
        self.assertTrue(th.compVarColTables("ms_sorted.ms", "mms_sorted.ms", "DATA"))
Example #20
0
 def test_partition_balanced_multiple_scan(self):
     partition(self.vis, outputvis=self.outputms, separationaxis='auto', numsubms=2, disableparallel=True)
     listpartition_dict = listpartition(self.outputms, createdict=True)
     self.assertEqual(len(listpartition_dict), 2)
     self.assertEqual(len(listpartition_dict[0]['scanId']), 26)
     self.assertEqual(len(listpartition_dict[1]['scanId']), 26)
     self.assertEqual(sum([x['nrows'] for x in listpartition_dict[1]['scanId'].values()]), 4344)
     self.assertEqual(sum([x['nrows'] for x in listpartition_dict[0]['scanId'].values()]), 4344)
Example #21
0
 def test_baseline_max(self):
     '''partition: create an MMS per baseline axis. Use the maximum number of baselines'''
     self.outputms = 'baseline_max.mms'
     partition(self.vis, outputvis=self.outputms, separationaxis='baseline', numsubms=50, flagbackup=False)
     
     # Take the dictionary and compare with original MS
     thisdict = listpartition(vis=self.outputms, createdict=True)
     self.assertEqual(len(thisdict.keys()), 6, 'There should be 6 subMSs in output MMS')
Example #22
0
 def test_baseline1(self):
     '''partition: create an MMS per baseline axis. Use the number of MPI servers'''
     self.outputms = 'baseline1.mms'
     partition(self.vis, outputvis=self.outputms, separationaxis='baseline', flagbackup=False)
     
     # The separation axis should be written to the output MMS
     sepaxis = ph.axisType(self.outputms)
     self.assertEqual(sepaxis, 'baseline', 'Partition did not write AxisType correctly in MMS')
Example #23
0
 def test_baseline_2subms(self):
     '''partition: create an MMS per baseline axis. Create 3 SubMS'''
     self.outputms = 'baseline_3subms.mms'
     partition(self.vis, outputvis=self.outputms, separationaxis='baseline', numsubms=3, flagbackup=False)
     
     # Take the dictionary and compare with original MS
     thisdict = listpartition(vis=self.outputms, createdict=True)
     self.assertEqual(len(thisdict.keys()), 3, 'There should be 3 subMSs in output MMS')
Example #24
0
 def test_partition_balanced_repeated_nsubms_greater_than_available_pairs(self):
     '''mstransform: Check that balanced mode properly handles the case when the number of requests
     subMSs is greater than the number of available ddi,scan pairs'''
     
     self.outputms = "test_partition_3c84scan1.mms"
     
     partition(self.vis, outputvis=self.outputms,separationaxis='auto',numsubms=64,flagbackup=False)
     listpartition_dict = listpartition(self.outputms, createdict=True)       
     self.assertEqual(len(listpartition_dict), 4, "Number of subMS should be 3")          
Example #25
0
    def test_default_scan(self):
        '''Partition: create an MMS with default values and axis=scan'''
        partition(vis=self.msfile,
                  outputvis=self.mmsfile,
                  separationaxis='scan')

        self.assertTrue(os.path.exists(self.mmsfile),
                        'MMS was not created for this test')

        # Take the dictionary and compare with original MS
        thisdict = listpartition(vis=self.mmsfile, createdict=True)

        # Compare nrows of all scans
        slist = ph.getMMSScans(thisdict)

        for s in slist:
            mmsN = ph.getMMSScanNrows(thisdict, s)
            msN = ph.getScanNrows(self.msfile, s)
            self.assertEqual(
                mmsN, msN,
                'Nrows in scan=%s differs: mms_nrows=%s <--> ms_nrows=%s' %
                (s, mmsN, msN))

        # Compare spw IDs
        for s in slist:
            mms_spw = ph.getSpwIds(self.mmsfile, s)
            ms_spw = ph.getSpwIds(self.msfile, s)
            self.assertEqual(mms_spw, ms_spw, 'list of spws in scan=%s differs: '\
                             'mms_spw=%s <--> ms_spw=%s' %(s, mmsN, msN))

        # Sort the output MSs so that they can be compared
        myms = mstool()

        myms.open(self.msfile)
        myms.sort('ms_sorted.ms', [
            'OBSERVATION_ID', 'ARRAY_ID', 'SCAN_NUMBER', 'FIELD_ID',
            'DATA_DESC_ID', 'ANTENNA1', 'ANTENNA2', 'TIME'
        ])
        myms.done()

        myms.open(self.mmsfile)
        myms.sort('mms_sorted.ms', [
            'OBSERVATION_ID', 'ARRAY_ID', 'SCAN_NUMBER', 'FIELD_ID',
            'DATA_DESC_ID', 'ANTENNA1', 'ANTENNA2', 'TIME'
        ])
        myms.done()

        self.assertTrue(
            th.compTables('ms_sorted.ms', 'mms_sorted.ms', [
                'FLAG', 'FLAG_CATEGORY', 'TIME_CENTROID', 'WEIGHT_SPECTRUM',
                'DATA'
            ]))

        # Compare the DATA column
        self.assertTrue(
            th.compVarColTables('ms_sorted.ms', 'mms_sorted.ms', 'DATA'))
Example #26
0
    def runPartition(self, ms, mmsdir, createlink, axis, subms):
        '''Run partition with default values to create an MMS.
           ms         --> full pathname of the MS
           mmsdir     --> directory to save the MMS to
           createlink --> when True, it will create a symbolic link to the
                         just created MMS in the same directory with extension .ms  
           axis      --> separationaxis to use (spw, scan, auto)
           subms  --> number of subMss to create

        '''
        from tasks import partition

        if not os.path.lexists(ms):
            return False
        
        # Create MMS name
        bname = os.path.basename(ms)
        if bname.endswith('.ms'):
            mmsname = bname.replace('.ms','.mms')
        else:
            mmsname = bname+'.mms'
        
        mms = os.path.join(self.mmsdir, mmsname)
        if os.path.lexists(mms):
            casalog.post('Output MMS already exist -->'+mms,'ERROR')
            return False
        
        # Check for remainings of corrupted mms
        corrupted = mms.replace('.mms','.data')
        if os.path.exists(corrupted):
            casalog.post('Cleaning up left overs','WARN')
            shutil.rmtree(corrupted)
        
        # Run partition   
        default('partition')
        partition(vis=ms, outputvis=mms, createmms=True, datacolumn='all', flagbackup=False,
                  separationaxis=axis, numsubms=subms)
        casalog.origin('convertToMMS')
        
        # Check if MMS was created
        if not os.path.exists(mms):
            casalog.post('Cannot create MMS ->'+mms, 'ERROR')
            return False
        
        # If requested, create a link to this MMS
        if createlink:
            here = os.getcwd()
            os.chdir(mmsdir)
            mmsname = os.path.basename(mms)
            lms = mmsname.replace('.mms', '.ms')
            casalog.post('Creating symbolic link to MMS')
            os.symlink(mmsname, lms)
            os.chdir(here)
                
        return True
Example #27
0
 def test_channels1(self):
     '''partition: create MMS with spw separation and channel selections'''
     partition(vis=self.msfile, outputvis=self.mmsfile, spw='0~4,5:1~10',createmms=True,
                 separationaxis='spw', disableparallel=True, flagbackup=False)
                         
     self.assertTrue(os.path.exists(self.mmsfile))
     
     # It should create 6 subMS, with spw=0~5
     # spw=5 should have only 10 channels
     ret = th.verifyMS(self.mmsfile, 6, 10, 5,ignoreflags=True)
     self.assertTrue(ret[0],ret[1])        
Example #28
0
 def test_mms_float(self):
     '''partition: '''
     partition(vis=self.msfile, outputvis=self.mmsfile,spw='1,3,5',
               datacolumn='FLOAT_DATA', disableparallel=True,flagbackup=False)
     
     ret = th.verifyMS(self.mmsfile, 3, 512, 0, [], ignoreflags=True)
     self.assertTrue(ret[0],ret[1])
     ret = th.verifyMS(self.mmsfile, 3, 512, 1, [], ignoreflags=True)
     self.assertTrue(ret[0],ret[1])
     ret = th.verifyMS(self.mmsfile, 3, 1024, 2, [], ignoreflags=True)
     self.assertTrue(ret[0],ret[1])
Example #29
0
 def test_partition_balanced_repeated_spws(self):
     '''mstransform: Check that balanced mode properly distributes the chunks when one SPW is
     pointed by more than one DDI (with different correlation setups)'''
     
     self.outputms = "test_partition_balanced_repeated_spws.mms"
     
     partition(self.vis, outputvis=self.outputms,separationaxis='auto',spw='0,2',numsubms=3,flagbackup=False)
     listpartition_dict = listpartition(self.outputms, createdict=True)
     self.assertEqual(listpartition_dict[0]['scanId'][1]['spwIds'], [1], "Wrong SPWIds in subMS 0")         
     self.assertEqual(listpartition_dict[1]['scanId'][1]['spwIds'], [0], "Wrong SPWIds in subMS 1")  
     self.assertEqual(listpartition_dict[2]['scanId'][1]['spwIds'], [0], "Wrong SPWIds in subMS 2")                      
Example #30
0
    def test_baseline1(self):
        '''partition: create an MMS per baseline axis. Use the number of MPI servers'''
        self.outputms = 'baseline1.mms'
        partition(self.vis,
                  outputvis=self.outputms,
                  separationaxis='baseline',
                  flagbackup=False)

        # The separation axis should be written to the output MMS
        sepaxis = ph.axisType(self.outputms)
        self.assertEqual(sepaxis, 'baseline',
                         'Partition did not write AxisType correctly in MMS')
Example #31
0
    def test_baseline_max(self):
        '''partition: create an MMS per baseline axis. Use the maximum number of baselines'''
        self.outputms = 'baseline_max.mms'
        partition(self.vis,
                  outputvis=self.outputms,
                  separationaxis='baseline',
                  numsubms=50,
                  flagbackup=False)

        # Take the dictionary and compare with original MS
        thisdict = listpartition(vis=self.outputms, createdict=True)
        self.assertEqual(len(thisdict.keys()), 6,
                         'There should be 6 subMSs in output MMS')
Example #32
0
 def test_baseline_autocorr(self):
     '''partition: create an MMS per baseline axis only for auto-correlations'''
     self.outputms = 'baseline_autocorr.mms'
     partition(self.vis, outputvis=self.outputms, separationaxis='baseline', antenna='*&&&', flagbackup=False)
     
     md = msmdtool()
     md.open(self.outputms)
     bsl = md.baselines()
     md.close()
     
     #diagnoals give the auto-corrs
     ac = bsl.diagonal()
     self.assertTrue(ac.all(), 'Not all auto-correlations are there')
Example #33
0
 def test_channels2(self):
     '''partition: create MMS with spw/scan separation and channel selections'''
     partition(vis=self.msfile, outputvis=self.mmsfile, spw='0:0~10,1:60~63',createmms=True,
                 separationaxis='auto', disableparallel=True, flagbackup=False, datacolumn='data')
                         
     self.assertTrue(os.path.exists(self.mmsfile))
     
     # It should create 4 subMS, with spw=0~1
     # spw=0 has 11 channels, spw=1 has 4 channels
     ret = th.verifyMS(self.mmsfile, 2, 11, 0, ignoreflags=True)
     self.assertTrue(ret[0],ret[1])        
     ret = th.verifyMS(self.mmsfile, 2, 4, 1, ignoreflags=True)
     self.assertTrue(ret[0],ret[1])        
Example #34
0
    def test_baseline_2subms(self):
        '''partition: create an MMS per baseline axis. Create 3 SubMS'''
        self.outputms = 'baseline_3subms.mms'
        partition(self.vis,
                  outputvis=self.outputms,
                  separationaxis='baseline',
                  numsubms=3,
                  flagbackup=False)

        # Take the dictionary and compare with original MS
        thisdict = listpartition(vis=self.outputms, createdict=True)
        self.assertEqual(len(thisdict.keys()), 3,
                         'There should be 3 subMSs in output MMS')
Example #35
0
    def test_default_scan(self):
        """Partition: create an MMS with default values and axis=scan"""
        partition(vis=self.msfile, outputvis=self.mmsfile, separationaxis="scan")

        self.assertTrue(os.path.exists(self.mmsfile), "MMS was not created for this test")

        # Take the dictionary and compare with original MS
        thisdict = listpartition(vis=self.mmsfile, createdict=True)

        # Compare nrows of all scans
        slist = ph.getMMSScans(thisdict)

        for s in slist:
            mmsN = ph.getMMSScanNrows(thisdict, s)
            msN = ph.getScanNrows(self.msfile, s)
            self.assertEqual(mmsN, msN, "Nrows in scan=%s differs: mms_nrows=%s <--> ms_nrows=%s" % (s, mmsN, msN))

        # Compare spw IDs
        for s in slist:
            mms_spw = ph.getSpwIds(self.mmsfile, s)
            ms_spw = ph.getSpwIds(self.msfile, s)
            self.assertEqual(
                mms_spw, ms_spw, "list of spws in scan=%s differs: " "mms_spw=%s <--> ms_spw=%s" % (s, mmsN, msN)
            )

        # Sort the output MSs so that they can be compared
        myms = mstool()

        myms.open(self.msfile)
        myms.sort(
            "ms_sorted.ms",
            ["OBSERVATION_ID", "ARRAY_ID", "SCAN_NUMBER", "FIELD_ID", "DATA_DESC_ID", "ANTENNA1", "ANTENNA2", "TIME"],
        )
        myms.done()

        myms.open(self.mmsfile)
        myms.sort(
            "mms_sorted.ms",
            ["OBSERVATION_ID", "ARRAY_ID", "SCAN_NUMBER", "FIELD_ID", "DATA_DESC_ID", "ANTENNA1", "ANTENNA2", "TIME"],
        )
        myms.done()

        self.assertTrue(
            th.compTables(
                "ms_sorted.ms", "mms_sorted.ms", ["FLAG", "FLAG_CATEGORY", "TIME_CENTROID", "WEIGHT_SPECTRUM", "DATA"]
            )
        )

        # Compare the DATA column
        self.assertTrue(th.compVarColTables("ms_sorted.ms", "mms_sorted.ms", "DATA"))
Example #36
0
 def test_linked_cols(self):
     '''partition: Verify that SYSPOWER, POINTING and SYSCAL are links'''
     self.outputms = "linkcols.mms"
     partition(self.vis, outputvis=self.outputms, numsubms=4, flagbackup=False)
     self.assertTrue(os.path.exists(self.outputms),'Output MMS does not exist')
     
     # Columns SYSPOWER, POINTING and SYSCAL should be links in most subMS
     mslocal = mstool()
     mslocal.open(self.outputms)
     subms = mslocal.getreferencedtables()
     mslocal.close()
     cols = ['POINTING','SYSCAL','SYSPOWER']
     for col in cols:
         self.assertTrue(os.path.islink(subms[1] + '/' + col))
Example #37
0
    def test_channels3(self):
        '''partition: verify spw sub-table consolidation'''
        partition(vis=self.msfile, outputvis=self.mmsfile, spw='3,5:10~19,7,9,11,13,15',
                    createmms=True,separationaxis='spw', flagbackup=False, datacolumn='data')       
                             
        self.assertTrue(os.path.exists(self.mmsfile))
        
        # spw=5 should be spw=1 after consolidation, with 10 channels
        ret = th.verifyMS(self.mmsfile, 7, 10, 1, ignoreflags=True)
        self.assertTrue(ret[0],ret[1])

        # The separation axis should be written to the output MMS
        sepaxis = ph.axisType(self.mmsfile)
        self.assertEqual(sepaxis, 'spw', 'Partition did not write AxisType correctly in MMS')
Example #38
0
 def createMMS(self, msfile, axis='auto',scans='',spws='', fields='', numms=64):
     '''Create MMSs for tests with input MMS'''
     prefix = msfile[:-len('.ms')]
     
     # Create an MMS for the tests
     self.testmms = "cvel_input_test.mms"
     default(partition)
     
     if os.path.exists(self.testmms):
         os.system("rm -rf " + self.testmms)
         os.system("rm -rf " + self.testmms +'.flagversions')
         
     print "................. Creating test MMS .................."
     partition(vis=msfile, outputvis=self.testmms,flagbackup=False, separationaxis=axis, scan=scans, spw=spws, field=fields,
               numsubms=numms)
Example #39
0
    def test_mms_float(self):
        '''partition: '''
        partition(vis=self.msfile,
                  outputvis=self.mmsfile,
                  spw='1,3,5',
                  datacolumn='FLOAT_DATA',
                  disableparallel=True,
                  flagbackup=False)

        ret = th.verifyMS(self.mmsfile, 3, 512, 0, [], ignoreflags=True)
        self.assertTrue(ret[0], ret[1])
        ret = th.verifyMS(self.mmsfile, 3, 512, 1, [], ignoreflags=True)
        self.assertTrue(ret[0], ret[1])
        ret = th.verifyMS(self.mmsfile, 3, 1024, 2, [], ignoreflags=True)
        self.assertTrue(ret[0], ret[1])
Example #40
0
    def test_partition_balanced_repeated_nsubms_greater_than_available_pairs(
            self):
        '''mstransform: Check that balanced mode properly handles the case when the number of requests
        subMSs is greater than the number of available ddi,scan pairs'''

        self.outputms = "test_partition_3c84scan1.mms"

        partition(self.vis,
                  outputvis=self.outputms,
                  separationaxis='auto',
                  numsubms=64,
                  flagbackup=False)
        listpartition_dict = listpartition(self.outputms, createdict=True)
        self.assertEqual(len(listpartition_dict), 4,
                         "Number of subMS should be 3")
Example #41
0
    def test_channels1(self):
        '''partition: create MMS with spw separation and channel selections'''
        partition(vis=self.msfile,
                  outputvis=self.mmsfile,
                  spw='0~4,5:1~10',
                  createmms=True,
                  separationaxis='spw',
                  disableparallel=True,
                  flagbackup=False)

        self.assertTrue(os.path.exists(self.mmsfile))

        # It should create 6 subMS, with spw=0~5
        # spw=5 should have only 10 channels
        ret = th.verifyMS(self.mmsfile, 6, 10, 5, ignoreflags=True)
        self.assertTrue(ret[0], ret[1])
Example #42
0
 def test_partition_balanced_multiple_scan(self):
     partition(self.vis,
               outputvis=self.outputms,
               separationaxis='auto',
               numsubms=2,
               disableparallel=True)
     listpartition_dict = listpartition(self.outputms, createdict=True)
     self.assertEqual(len(listpartition_dict), 2)
     self.assertEqual(len(listpartition_dict[0]['scanId']), 26)
     self.assertEqual(len(listpartition_dict[1]['scanId']), 26)
     self.assertEqual(
         sum([x['nrows']
              for x in listpartition_dict[1]['scanId'].values()]), 4344)
     self.assertEqual(
         sum([x['nrows']
              for x in listpartition_dict[0]['scanId'].values()]), 4344)
Example #43
0
 def createMMS(self, msfile, column='data', axis='auto',scans='',spws=''):
     '''Create MMSs for tests with input MMS'''
     prefix = msfile.rstrip('.ms')
     if not os.path.exists(msfile):
         os.system('cp -RL '+datapath + msfile +' '+ msfile)
     
     # Create an MMS for the tests
     self.testmms = prefix + ".test.mms"
     default(mstransform)
     
     if os.path.exists(self.testmms):
         os.system("rm -rf " + self.testmms)
         
     print "................. Creating test MMS .................."
     partition(vis=msfile, outputvis=self.testmms, datacolumn=column,
                 createmms=True,separationaxis=axis, scan=scans, spw=spws)
Example #44
0
    def test_baseline_autocorr(self):
        '''partition: create an MMS per baseline axis only for auto-correlations'''
        self.outputms = 'baseline_autocorr.mms'
        partition(self.vis,
                  outputvis=self.outputms,
                  separationaxis='baseline',
                  antenna='*&&&',
                  flagbackup=False)

        md = msmdtool()
        md.open(self.outputms)
        bsl = md.baselines()
        md.close()

        #diagnoals give the auto-corrs
        ac = bsl.diagonal()
        self.assertTrue(ac.all(), 'Not all auto-correlations are there')
Example #45
0
    def test_scan_spw(self):
        '''Partition: separationaxis=scan with spw selection'''
        partition(vis=self.msfile,
                  outputvis=self.mmsfile,
                  separationaxis='scan',
                  spw='1~4,10,11',
                  flagbackup=False,
                  datacolumn='data')

        self.assertTrue(os.path.exists(self.mmsfile),
                        'MMS was not created for this test')
        self.assertTrue(
            os.path.exists(self.msfile),
            'Make sure the input MS is not deleted inside the task')

        # Take the dictionary and compare with original MS
        thisdict = listpartition(vis=self.mmsfile, createdict=True)

        # Dictionary with selection to compare with original MS
        mysel = {'spw': '1~4,10,11'}

        # Compare nrows of all scans in selection
        slist = ph.getMMSScans(thisdict)
        for s in slist:
            mmsN = ph.getMMSScanNrows(thisdict, s)
            msN = ph.getScanNrows(self.msfile, s, selection=mysel)
            self.assertEqual(
                mmsN, msN,
                'Nrows in scan=%s differs: mms_nrows=%s <--> ms_nrows=%s' %
                (s, mmsN, msN))

        # The comparison should be
        # ms_spw = 1 --> mms_spw = 0
        # ms_spw = 2 --> mms_spw = 1, etc.
        # Check that MMS spw IDs have been re-indexed properly
        indexed_ids = range(6)
        for s in slist:
            mms_spw = ph.getSpwIds(self.mmsfile, s)
            self.assertEqual(mms_spw, indexed_ids,
                             'spw IDs were not properly re-indexed')

        # The separation axis should be written to the output MMS
        sepaxis = ph.axisType(self.mmsfile)
        self.assertEqual(sepaxis, 'scan',
                         'Partition did not write AxisType correctly in MMS')
Example #46
0
    def test_linked_cols(self):
        '''partition: Verify that SYSPOWER, POINTING and SYSCAL are links'''
        self.outputms = "linkcols.mms"
        partition(self.vis,
                  outputvis=self.outputms,
                  numsubms=4,
                  flagbackup=False)
        self.assertTrue(os.path.exists(self.outputms),
                        'Output MMS does not exist')

        # Columns SYSPOWER, POINTING and SYSCAL should be links in most subMS
        mslocal = mstool()
        mslocal.open(self.outputms)
        subms = mslocal.getreferencedtables()
        mslocal.close()
        cols = ['POINTING', 'SYSCAL', 'SYSPOWER']
        for col in cols:
            self.assertTrue(os.path.islink(subms[1] + '/' + col))
Example #47
0
 def test_baseline_partition(self):
     '''partition: create an MMS per baseline for an interferometry MS'''
     partition(vis=self.msfile, outputvis=self.mmsfile, spw='0,1',createmms=True,
               separationaxis='baseline', flagbackup=False, datacolumn='data')  
     
     listpartition(self.mmsfile) 
     
     # This MS doesn't have auto-correlations. It has 4 antennas and 6 baselines (cross)
     # There will be MSSelectionNullSelection SEVERE messages in the logfile because
     # it will try to create SubMSs from auto-correlations. This is not an error!
     md = msmdtool()
     md.open(self.mmsfile)
     bsl = md.baselines()
     md.close()
     
     # diagonal give the auto-corrs
     ac = bsl.diagonal()
     self.assertFalse(ac.all(), 'There should be no auto-correlations in MMS')
Example #48
0
    def test_split_float(self):
        """partition: split an MS with FLOAT_DATA"""
        partition(
            vis=self.msfile,
            outputvis=self.mmsfile,
            spw="1,3,5",
            createmms=False,
            datacolumn="FLOAT_DATA",
            disableparallel=True,
            flagbackup=False,
        )

        ret = th.verifyMS(self.mmsfile, 3, 512, 0, [], ignoreflags=True)
        self.assertTrue(ret[0], ret[1])
        ret = th.verifyMS(self.mmsfile, 3, 512, 1, [], ignoreflags=True)
        self.assertTrue(ret[0], ret[1])
        ret = th.verifyMS(self.mmsfile, 3, 1024, 2, [], ignoreflags=True)
        self.assertTrue(ret[0], ret[1])
Example #49
0
    def test_sepaxis(self):
        '''Partition: separationaxis=auto'''
        partition(vis=self.msfile,
                  outputvis=self.mmsfile,
                  spw='0~11',
                  separationaxis='auto',
                  flagbackup=False,
                  disableparallel=True,
                  datacolumn='data')
        #        partition(vis=self.msfile, outputvis=self.mmsfile,separationaxis='auto')

        self.assertTrue(os.path.exists(self.mmsfile),
                        'MMS was not created for this test')

        # Dictionary with selection to compare with original MS
        mysel = {'spw': '0~11'}

        # Take the dictionary and compare with original MS
        thisdict = listpartition(vis=self.mmsfile, createdict=True)

        # Compare nrows of all scans in selection
        slist = ph.getMMSScans(thisdict)
        self.assertEqual(slist.__len__(), 2)
        for s in slist:
            mmsN = ph.getMMSScanNrows(thisdict, s)
            msN = ph.getScanNrows(self.msfile, s, selection=mysel)
            #            msN = ph.getScanNrows(self.msfile, s)
            self.assertEqual(
                mmsN, msN,
                'Nrows in scan=%s differs: mms_nrows=%s <--> ms_nrows=%s' %
                (s, mmsN, msN))

        # Compare spw IDs
        for s in slist:
            mms_spw = ph.getSpwIds(self.mmsfile, s)
            ms_spw = ph.getSpwIds(self.msfile, s, selection=mysel)
            #            ms_spw = ph.getSpwIds(self.msfile, s)
            self.assertEqual(mms_spw, ms_spw, 'list of spws in scan=%s differs: '\
                             'mms_spw=%s <--> ms_spw=%s' %(s, mms_spw, ms_spw))

        # The separation axis should be written to the output MMS
        sepaxis = ph.axisType(self.mmsfile)
        self.assertEqual(sepaxis, 'scan,spw',
                         'Partition did not write AxisType correctly in MMS')
Example #50
0
    def test_partition_balanced_repeated_spws(self):
        '''mstransform: Check that balanced mode properly distributes the chunks when one SPW is
        pointed by more than one DDI (with different correlation setups)'''

        self.outputms = "test_partition_balanced_repeated_spws.mms"

        partition(self.vis,
                  outputvis=self.outputms,
                  separationaxis='auto',
                  spw='0,2',
                  numsubms=3,
                  flagbackup=False)
        listpartition_dict = listpartition(self.outputms, createdict=True)
        self.assertEqual(listpartition_dict[0]['scanId'][1]['spwIds'], [1],
                         "Wrong SPWIds in subMS 0")
        self.assertEqual(listpartition_dict[1]['scanId'][1]['spwIds'], [0],
                         "Wrong SPWIds in subMS 1")
        self.assertEqual(listpartition_dict[2]['scanId'][1]['spwIds'], [0],
                         "Wrong SPWIds in subMS 2")
Example #51
0
    def test_numsubms(self):
        '''Partition: small numsubms value'''
        # There are 16 spws; we want only 6 sub-MSs.
        partition(vis=self.msfile,
                  outputvis=self.mmsfile,
                  separationaxis='spw',
                  numsubms=6,
                  flagbackup=False,
                  disableparallel=True,
                  datacolumn='corrected')

        self.assertTrue(os.path.exists(self.mmsfile),
                        'MMS was not created for this test')

        # Take the dictionary and compare with original MS
        thisdict = listpartition(vis=self.mmsfile, createdict=True)

        # Check the number of sub-MSs
        mmslist = []
        klist = thisdict.keys()
        for kk in klist:
            mmslist.append(thisdict[kk]['MS'])

        nsubms = mmslist.__len__()
        self.assertEqual(nsubms, 6, 'There should be only 6 sub-MSs')

        # Check that spw list is the same in MS and MMS
        spwlist = ph.getMMSSpwIds(thisdict)

        # Reference list of scans in MS
        slist = ph.getScanList(self.msfile)
        setlist = set([])
        for s in slist:
            ms_spw = ph.getSpwIds(self.msfile, s)
            for a in ms_spw:
                setlist.add(a)

        self.assertEqual(list(setlist), spwlist)

        # The separation axis should be written to the output MMS
        sepaxis = ph.axisType(self.mmsfile)
        self.assertEqual(sepaxis, 'spw',
                         'Partition did not write AxisType correctly in MMS')
Example #52
0
    def test_channels2(self):
        '''partition: create MMS with spw/scan separation and channel selections'''
        partition(vis=self.msfile,
                  outputvis=self.mmsfile,
                  spw='0:0~10,1:60~63',
                  createmms=True,
                  separationaxis='auto',
                  disableparallel=True,
                  flagbackup=False,
                  datacolumn='data')

        self.assertTrue(os.path.exists(self.mmsfile))

        # It should create 4 subMS, with spw=0~1
        # spw=0 has 11 channels, spw=1 has 4 channels
        ret = th.verifyMS(self.mmsfile, 2, 11, 0, ignoreflags=True)
        self.assertTrue(ret[0], ret[1])
        ret = th.verifyMS(self.mmsfile, 2, 4, 1, ignoreflags=True)
        self.assertTrue(ret[0], ret[1])
Example #53
0
    def test_channels3(self):
        '''partition: verify spw sub-table consolidation'''
        partition(vis=self.msfile,
                  outputvis=self.mmsfile,
                  spw='3,5:10~19,7,9,11,13,15',
                  createmms=True,
                  separationaxis='spw',
                  flagbackup=False,
                  datacolumn='data')

        self.assertTrue(os.path.exists(self.mmsfile))

        # spw=5 should be spw=1 after consolidation, with 10 channels
        ret = th.verifyMS(self.mmsfile, 7, 10, 1, ignoreflags=True)
        self.assertTrue(ret[0], ret[1])

        # The separation axis should be written to the output MMS
        sepaxis = ph.axisType(self.mmsfile)
        self.assertEqual(sepaxis, 'spw',
                         'Partition did not write AxisType correctly in MMS')
Example #54
0
    def createMMS(self, msfile, column='data', axis='auto', scans='', spws=''):
        '''Create MMSs for tests with input MMS'''
        prefix = msfile.rstrip('.ms')
        if not os.path.exists(msfile):
            os.system('cp -RL ' + datapath + msfile + ' ' + msfile)

        # Create an MMS for the tests
        self.testmms = prefix + ".test.mms"
        default(mstransform)

        if os.path.exists(self.testmms):
            os.system("rm -rf " + self.testmms)

        print "................. Creating test MMS .................."
        partition(vis=msfile,
                  outputvis=self.testmms,
                  datacolumn=column,
                  createmms=True,
                  separationaxis=axis,
                  scan=scans,
                  spw=spws)
Example #55
0
    def test_spw_separation(self):
        '''Partition: create an MMS separated by spws with observation selection'''
        # NOTE: ms.getscansummary() used in ph.getScanNrows does not honour several observation
        #       IDs, therefore I need to selection by obs id in partition
        partition(vis=self.msfile,
                  outputvis=self.mmsfile,
                  separationaxis='spw',
                  observation='2',
                  flagbackup=False,
                  disableparallel=True)

        self.assertTrue(os.path.exists(self.mmsfile),
                        'MMS was not created for this test')

        # Take the dictionary and compare with original MS
        thisdict = listpartition(vis=self.mmsfile, createdict=True)

        # Dictionary with selection to compare with original MS
        mysel = {'observation': '2'}

        # Compare nrows of all scans in MMS and MS
        slist = ph.getMMSScans(thisdict)
        for s in slist:
            mmsN = ph.getMMSScanNrows(thisdict, s)
            msN = ph.getScanNrows(self.msfile, s, selection=mysel)
            self.assertEqual(
                mmsN, msN,
                'Nrows in scan=%s differs: mms_nrows=%s <--> ms_nrows=%s' %
                (s, mmsN, msN))

        # spwids are re-indexed. The expected IDs are:
        # ms_spw = 2 --> mms_spw = 0
        # ms_spw = 3 --> mms_spw = 1, etc.
        # Check that MMS spw IDs have been re-indexed properly
        indexed_ids = range(4)
        for s in slist:
            mms_spw = ph.getSpwIds(self.mmsfile, s)
            self.assertEqual(mms_spw, indexed_ids,
                             'spw IDs were not properly re-indexed')
Example #56
0
    def test_spw_selection(self):
        '''Partition: create an MMS separated by spws with spw=2,4 selection'''
        partition(vis=self.msfile,
                  outputvis=self.mmsfile,
                  separationaxis='spw',
                  spw='2,4',
                  flagbackup=False,
                  disableparallel=True)

        self.assertTrue(os.path.exists(self.mmsfile),
                        'MMS was not created for this test')

        # Take the dictionary and compare with original MS
        thisdict = listpartition(vis=self.mmsfile, createdict=True)

        # Dictionary with selection to compare with original MS
        mysel = {'spw': '2,4'}

        # Compare nrows of all scans in selection
        slist = ph.getMMSScans(thisdict)
        for s in slist:
            mmsN = ph.getMMSScanNrows(thisdict, s)
            msN = ph.getScanNrows(self.msfile, s, selection=mysel)
            self.assertEqual(
                mmsN, msN,
                'Nrows in scan=%s differs: mms_nrows=%s <--> ms_nrows=%s' %
                (s, mmsN, msN))

        # spwids are re-indexed. The expected IDs are:
        # ms_spw = 2 --> mms_spw = 0
        # ms_spw = 4 --> mms_spw = 1
        # Check that MMS spw IDs have been re-indexed properly
        indexed_ids = range(2)
        for s in slist:
            mms_spw = ph.getSpwIds(self.mmsfile, s)
            self.assertEqual(mms_spw, indexed_ids,
                             'spw IDs were not properly re-indexed')
Example #57
0
    def test_baseline_partition(self):
        '''partition: create an MMS per baseline for an interferometry MS'''
        partition(vis=self.msfile,
                  outputvis=self.mmsfile,
                  spw='0,1',
                  createmms=True,
                  separationaxis='baseline',
                  flagbackup=False,
                  datacolumn='data')

        listpartition(self.mmsfile)

        # This MS doesn't have auto-correlations. It has 4 antennas and 6 baselines (cross)
        # There will be MSSelectionNullSelection SEVERE messages in the logfile because
        # it will try to create SubMSs from auto-correlations. This is not an error!
        md = msmdtool()
        md.open(self.mmsfile)
        bsl = md.baselines()
        md.close()

        # diagonal give the auto-corrs
        ac = bsl.diagonal()
        self.assertFalse(ac.all(),
                         'There should be no auto-correlations in MMS')
Example #58
0
    def test_merge_subtables_after_partiton_evla(self):
        '''mstransform: Check that sub-tables are properly merged after partiton'''

        self.outputms = "parted-evla.ms"

        partition(self.vis,
                  outputvis=self.outputms,
                  separationaxis='spw',
                  numsubms=4,
                  flagbackup=False)

        subtable = "/FEED"
        sort_order = "SPECTRAL_WINDOW_ID, ANTENNA_ID, FEED_ID, TIME"
        res = th.compareSubTables(self.outputms + subtable,
                                  self.vis + subtable, sort_order)
        self.assertTrue(
            res[0], "Error comparing " + res[1] + " column from " + subtable +
            " sub-table")

        subtable = "/CALDEVICE"
        sort_order = "SPECTRAL_WINDOW_ID, ANTENNA_ID, FEED_ID, TIME"
        excluded_cols = [
            'CAL_LOAD_NAMES', 'NOISE_CAL', 'CAL_EFF', 'TEMPERATURE_LOAD'
        ]
        res = th.compareSubTables(self.outputms + subtable,
                                  self.vis + subtable, sort_order,
                                  excluded_cols)
        self.assertTrue(
            res[0], "Error comparing " + res[1] + " column from " + subtable +
            " sub-table")

        subtable = "/SYSPOWER"
        sort_order = "SPECTRAL_WINDOW_ID, ANTENNA_ID, FEED_ID, TIME"
        excluded_cols = [
            'SWITCHED_DIFF', 'SWITCHED_SUM', 'REQUANTIZER_GAIN',
            'TANT_SPECTRUM'
        ]
        res = th.compareSubTables(self.outputms + subtable,
                                  self.vis + subtable, sort_order,
                                  excluded_cols)
        self.assertTrue(
            res[0], "Error comparing " + res[1] + " column from " + subtable +
            " sub-table")

        subtable = "/SOURCE"
        sort_order = "SPECTRAL_WINDOW_ID, TIME, SOURCE_ID"
        excluded_cols = ['POSITION', 'TRANSITION', 'REST_FREQUENCY', 'SYSVEL']
        res = th.compareSubTables(self.outputms + subtable,
                                  self.vis + subtable, sort_order,
                                  excluded_cols)
        self.assertTrue(
            res[0], "Error comparing " + res[1] + " column from " + subtable +
            " sub-table")

        subtable = "/SPECTRAL_WINDOW"
        excluded_cols = ['ASSOC_SPW_ID', 'ASSOC_NATURE']
        res = th.compareSubTables(self.outputms + subtable,
                                  self.vis + subtable, None, excluded_cols)
        self.assertTrue(
            res[0], "Error comparing " + res[1] + " column from " + subtable +
            " sub-table")
Example #59
0
def main(thislist, axis='auto', numsubms=4):

    if thislist == []:
        print 'Need list of tasks to run.'
        usage()
        os._exit(0)

    print "Will create MMS for the following tasks %s" % thislist
    print

    # Loop through task list
    for t in thislist:
        if t not in TASKLIST:
            print 'ERROR: task ' + t + ' is not in TASKLIST. Run this script with -l for the full list.'
            os._exit(0)

#       if t == 'flagdata':


#            axis='scan'

        mmstest(t, axis, numsubms)

    from tasks import partition, importuvfits

    #    if 'listvis' in thislist:
    #        # NOTE for test_listvis data:
    #        # You need to run partition by hand to create an MMS for the single-dish data set
    #        SDPATH = DATAPATH + 'unittest/listvis/'
    #        SDMMS = './unittest_mms/listvis/'
    #
    #        partition(vis=SDPATH+'OrionS_rawACSmod', outputvis=SDMMS+'OrionS_rawACSmod.mms',
    #                  datacolumn='float_data', createmms=True, flagbackup=False)

    if 'split' in thislist:
        # some additional MMSs
        SPLITMMSPATH = './unittest_mms/split/'
        specialcase = [
            '0420+417/0420+417.ms', 'viewertest/ctb80-vsm.ms',
            'split/labelled_by_time+ichan.ms'
        ]
        for myms in specialcase:
            shutil.rmtree(SPLITMMSPATH + os.path.basename(myms),
                          ignore_errors=True)
            partition(vis=DATAPATH + myms,
                      outputvis=SPLITMMSPATH + os.path.basename(myms),
                      datacolumn='all',
                      flagbackup=False)

        # workaround for a partition shortcoming: column keywords not copied
        tb.open(SPLITMMSPATH + 'hasfc.mms/SUBMSS/hasfc.0000.ms/',
                nomodify=False)
        tb.putcolkeyword('FLAG_CATEGORY', 'CATEGORY',
                         ['FLAG_CMD', 'ORIGINAL', 'USER'])
        tb.close()

    if 'wvrgcal' in thislist:
        WVRGCALMMSPATH = './unittest_mms/wvrgcal/'
        WVRGCALPATH = DATAPATH + 'unittest/wvrgcal/input/'
        origwd = os.getcwd()
        os.chdir(WVRGCALMMSPATH)
        shutil.rmtree('input', ignore_errors=True)
        os.mkdir('input')
        os.chdir('input')
        mydirs = os.listdir(WVRGCALPATH)
        for d in mydirs:
            print d
            if os.path.splitext(d)[1] == '.ms':
                partition(vis=WVRGCALPATH + d,
                          outputvis=d,
                          datacolumn='all',
                          numsubms=5,
                          flagbackup=False)
            else:
                os.symlink(WVRGCALPATH + d, d)
        os.chdir(origwd)

    if ('concat' in thislist):
        CONCATMMSPATH = './unittest_mms/concat/'
        CONCATPATH = DATAPATH + 'unittest/concat/input/'
        origwd = os.getcwd()
        os.chdir(CONCATMMSPATH)
        shutil.rmtree('input', ignore_errors=True)
        os.mkdir('input')
        os.chdir('input')
        mydirs = os.listdir(CONCATPATH)
        for d in mydirs:
            print d
            if os.path.splitext(d)[1] == '.ms':
                partition(vis=CONCATPATH + d,
                          outputvis=d,
                          datacolumn='all',
                          numsubms=6,
                          flagbackup=False)
            else:
                os.symlink(CONCATPATH + d, d)
        os.chdir(origwd)

    if ('cvel' in thislist):

        CVELPATH = DATAPATH + 'ngc4826/fitsfiles/'
        MMSPATH = './unittest_mms/cvel/'
        mmsdir = MMSPATH + 'ngc4826.mms'
        tempdir = 'makemmsdirtemp'
        os.system('mkdir ' + tempdir)
        importuvfits(fitsfile=CVELPATH + 'ngc4826.ll.fits5',
                     vis=tempdir + '/ngc4826.ms')
        partition(vis=tempdir + '/ngc4826.ms',
                  outputvis=MMSPATH + 'ngc4826.mms',
                  separationaxis='scan',
                  flagbackup=False,
                  datacolumn='all')
        os.system('rm -rf ' + tempdir)

        CVELPATH = DATAPATH + 'cvel/input/'
        cvelfiles = [
            'jupiter6cm.demo-thinned.ms',
            'g19_d2usb_targets_line-shortened-thinned.ms',
            'evla-highres-sample-thinned.ms'
        ]
        MMSPATH = './unittest_mms/cvel/'
        thisdir = os.getcwd()
        for cvelms in cvelfiles:
            mmsname = cvelms.replace('.ms', '.mms')
            partition(vis=CVELPATH + cvelms,
                      outputvis=MMSPATH + mmsname,
                      separationaxis='scan',
                      flagbackup=False,
                      datacolumn='all',
                      numsubms=4)
            os.chdir(MMSPATH)
            os.system('ln -s ' + mmsname + ' ' + cvelms)
            os.chdir(thisdir)

        # Create the jup.mms file
        mmsname = 'jup.mms'
        output = MMSPATH + mmsname
        split2(vis=MMSPATH + '/jupiter6cm.demo-thinned.mms',
               outputvis=output,
               field='JUPITER',
               datacolumn='data')
        tblocal = tbtool()
        tblocal.open(output, nomodify=False)
        a = tblocal.getcol('TIME')
        delta = (54709. * 86400 - a[0])
        a = a + delta
        strt = a[0]
        tblocal.putcol('TIME', a)
        a = tblocal.getcol('TIME_CENTROID')
        a = a + delta
        tblocal.putcol('TIME_CENTROID', a)
        tblocal.close()
        tblocal.open(output + '/OBSERVATION', nomodify=False)
        a = tblocal.getcol('TIME_RANGE')
        delta = strt - a[0][0]
        a = a + delta
        tblocal.putcol('TIME_RANGE', a)
        tblocal.close()
        tblocal.open(output + '/FIELD', nomodify=False)
        a = tblocal.getcol('TIME')
        delta = strt - a[0]
        a = a + delta
        tblocal.putcol('TIME', a)
        tblocal.close()
        mslocal = mstool()
        mslocal.open(output, nomodify=False)
        mslocal.addephemeris(
            0,
            os.environ.get('CASAPATH').split()[0] +
            '/data/ephemerides/JPL-Horizons/Jupiter_54708-55437dUTC.tab',
            'Jupiter_54708-55437dUTC', 0)
        mslocal.close()

        CVELMS = DATAPATH + 'fits-import-export/input/test.ms'
        MMSPATH = './unittest_mms/cvel/'
        thisdir = os.getcwd()
        partition(vis=CVELMS,
                  outputvis=MMSPATH + 'test.mms',
                  separationaxis='scan',
                  flagbackup=False,
                  datacolumn='all',
                  numsubms=4)
        os.chdir(MMSPATH)
        os.system('ln -s test.mms test.ms')
        os.chdir(thisdir)

    if ('fixvis' in thislist):
        MSPATH = os.environ.get(
            'CASAPATH').split()[0] + '/data/regression/0420+417/'
        MSNAME = MSPATH + '0420+417.ms'
        MMSPATH = './unittest_mms/fixvis/'
        MMSNAME = MMSPATH + '0420+417.mms'
        partition(vis=MSNAME,
                  outputvis=MMSNAME,
                  datacolumn='all',
                  separationaxis=axis,
                  numsubms=numsubms,
                  flagbackup=False)
        # Create symlink
        thisdir = os.getcwd()
        os.chdir(MMSPATH)
        os.system('ln -s 0420+417.mms 0420+417.ms')
        os.chdir(thisdir)
Example #60
0
    def test_default(self):
        '''Partition: create an MMS with default values in parallel'''

        # First split off one scan to run the test faster
        split(vis=self.msfile,
              outputvis='split30.ms',
              datacolumn='DATA',
              scan='30')
        msfile = 'split30.ms'

        partition(vis=msfile, outputvis=self.mmsfile)

        self.assertTrue(os.path.exists(self.mmsfile),
                        'MMS was not created for this test')

        # Gather several metadata information
        # for the MS
        mdlocal1 = msmdtool()
        mdlocal1.open(msfile)
        ms_rows = mdlocal1.nrows()
        ms_nscans = mdlocal1.nscans()
        ms_nspws = mdlocal1.nspw()
        ms_scans = mdlocal1.scannumbers()
        mdlocal1.close()

        # for the MMS
        mdlocal2 = msmdtool()
        mdlocal2.open(self.mmsfile)
        mms_rows = mdlocal2.nrows()
        mms_nscans = mdlocal2.nscans()
        mms_nspws = mdlocal2.nspw()
        mms_scans = mdlocal2.scannumbers()
        mdlocal2.close()

        # Compare the number of rows
        self.assertEqual(ms_rows, mms_rows,
                         'Compare total number of rows in MS and MMS')
        self.assertEqual(ms_nscans, mms_nscans, 'Compare number of scans')
        self.assertEqual(ms_nspws, mms_nspws, 'Compare number of spws')

        # Compare the scans
        self.assertEqual(ms_scans.all(), mms_scans.all(),
                         'Compare all scan IDs')

        try:
            mdlocal1.open(msfile)
            mdlocal2.open(self.mmsfile)

            # Compare the spws
            for i in ms_scans:
                msi = mdlocal1.spwsforscan(i)
                mmsi = mdlocal2.spwsforscan(i)
                self.assertEqual(msi.all(), mmsi.all(),
                                 'Compare spw Ids for a scan')
        finally:
            mdlocal1.close()
            mdlocal2.close()

        # Sort the output MSs so that they can be compared
        myms = mstool()

        myms.open(msfile)
        myms.sort('ms_sorted.ms', [
            'OBSERVATION_ID', 'ARRAY_ID', 'SCAN_NUMBER', 'FIELD_ID',
            'DATA_DESC_ID', 'ANTENNA1', 'ANTENNA2', 'TIME'
        ])
        myms.done()

        myms.open(self.mmsfile)
        myms.sort('mms_sorted.ms', [
            'OBSERVATION_ID', 'ARRAY_ID', 'SCAN_NUMBER', 'FIELD_ID',
            'DATA_DESC_ID', 'ANTENNA1', 'ANTENNA2', 'TIME'
        ])
        myms.done()

        # Ignore WEIGHT_SPECTRUM and SIGMA_SPECTRUM, which are empty columns
        self.assertTrue(
            th.compTables('ms_sorted.ms', 'mms_sorted.ms', [
                'FLAG', 'FLAG_CATEGORY', 'TIME_CENTROID', 'WEIGHT_SPECTRUM',
                'SIGMA_SPECTRUM', 'DATA'
            ]))

        # Compare the DATA column
        self.assertTrue(
            th.compVarColTables('ms_sorted.ms', 'mms_sorted.ms', 'DATA'))

        # The separation axis should be written to the output MMS
        sepaxis = ph.axisType(self.mmsfile)
        self.assertEqual(sepaxis, 'scan,spw',
                         'Partition did not write AxisType correctly in MMS')