def test_freqavg9(self):
'''mstranform: Average using different bins and a channel selection, output MMS'''
self.outputms = "favg9.ms"
mstransform(vis=self.vis, outputvis=self.outputms, spw='2,12,10:1~10', chanaverage=True,
chanbin=[32,128,5], createmms=True, separationaxis='spw')
self.assertTrue(os.path.exists(self.outputms))
# Output should be:
# spw=0 4 channels
# spw=1 1 channel
# spw=2 2 channels
ret = th.verifyMS(self.outputms, 3, 4, 0, ignoreflags=True)
self.assertTrue(ret[0],ret[1])
ret = th.verifyMS(self.outputms, 3, 1, 1, ignoreflags=True)
self.assertTrue(ret[0],ret[1])
ret = th.verifyMS(self.outputms, 3, 2, 2, ignoreflags=True)
self.assertTrue(ret[0],ret[1])
# Verify that some sub-tables are properly re-indexed.
spw_col = th.getVarCol(self.outputms+'/DATA_DESCRIPTION', 'SPECTRAL_WINDOW_ID')
self.assertEqual(spw_col.keys().__len__(), 3, 'Wrong number of rows in DD table')
self.assertEqual(spw_col['r1'][0], 0,'Error re-indexing DATA_DESCRIPTION table')
self.assertEqual(spw_col['r2'][0], 1,'Error re-indexing DATA_DESCRIPTION table')
self.assertEqual(spw_col['r3'][0], 2,'Error re-indexing DATA_DESCRIPTION table')
def test_freqavg7(self):
'''mstranform: Average using different bins for several spws, output MMS'''
# same as test_freqavg4
self.outputms = "favg7.ms"
mstransform(vis=self.vis, outputvis=self.outputms, spw='10,12,20', chanaverage=True,
chanbin=[128,4,10], createmms=True, separationaxis='scan', disableparallel=True)
self.assertTrue(os.path.exists(self.outputms))
# Output should be:
# spw=0 1 channel
# spw=1 32 channels
# spw=3 13 channels
ret = th.verifyMS(self.outputms, 3, 1, 0, ignoreflags=True)
self.assertTrue(ret[0],ret[1])
ret = th.verifyMS(self.outputms, 3, 32, 1, ignoreflags=True)
self.assertTrue(ret[0],ret[1])
ret = th.verifyMS(self.outputms, 3, 12, 2, ignoreflags=True)
self.assertTrue(ret[0],ret[1])
# Verify that some sub-tables are properly re-indexed.
spw_col = th.getVarCol(self.outputms+'/DATA_DESCRIPTION', 'SPECTRAL_WINDOW_ID')
self.assertEqual(spw_col.keys().__len__(), 3, 'Wrong number of rows in DD table')
self.assertEqual(spw_col['r1'][0], 0,'Error re-indexing DATA_DESCRIPTION table')
self.assertEqual(spw_col['r2'][0], 1,'Error re-indexing DATA_DESCRIPTION table')
self.assertEqual(spw_col['r3'][0], 2,'Error re-indexing DATA_DESCRIPTION table')
def test_freqavg10(self):
'''mstranform: Average using different bins, channel selection, both axes, output MMS'''
self.outputms = "favg10.ms"
mstransform(vis=self.vis, outputvis=self.outputms, spw='2,12,10:1~10', chanaverage=True,
chanbin=[32,128,5], createmms=True, separationaxis='auto', numsubms=6)
self.assertTrue(os.path.exists(self.outputms))
# Should create 6 subMSs
mslocal = mstool()
mslocal.open(thems=self.outputms)
sublist = mslocal.getreferencedtables()
mslocal.close()
self.assertEqual(sublist.__len__(), 6, 'Should have created 6 subMSs')
# Output should be:
# spw=0 4 channels
# spw=1 1 channel
# spw=2 2 channels
ret = th.verifyMS(self.outputms, 3, 4, 0, ignoreflags=True)
self.assertTrue(ret[0],ret[1])
ret = th.verifyMS(self.outputms, 3, 1, 1, ignoreflags=True)
self.assertTrue(ret[0],ret[1])
ret = th.verifyMS(self.outputms, 3, 2, 2, ignoreflags=True)
self.assertTrue(ret[0],ret[1])
# Verify that some sub-tables are properly re-indexed.
spw_col = th.getVarCol(self.outputms+'/DATA_DESCRIPTION', 'SPECTRAL_WINDOW_ID')
self.assertEqual(spw_col.keys().__len__(), 3, 'Wrong number of rows in DD table')
self.assertEqual(spw_col['r1'][0], 0,'Error re-indexing DATA_DESCRIPTION table')
self.assertEqual(spw_col['r2'][0], 1,'Error re-indexing DATA_DESCRIPTION table')
self.assertEqual(spw_col['r3'][0], 2,'Error re-indexing DATA_DESCRIPTION table')
def test_mms_float(self):
'''partition: '''
partition(vis=self.msfile, outputvis=self.mmsfile,spw='1,3,5',
datacolumn='FLOAT_DATA', disableparallel=True,flagbackup=False)
ret = th.verifyMS(self.mmsfile, 3, 512, 0, [], ignoreflags=True)
self.assertTrue(ret[0],ret[1])
ret = th.verifyMS(self.mmsfile, 3, 512, 1, [], ignoreflags=True)
self.assertTrue(ret[0],ret[1])
ret = th.verifyMS(self.mmsfile, 3, 1024, 2, [], ignoreflags=True)
self.assertTrue(ret[0],ret[1])
def test_parallel3(self):
'''mstransform: create MMS with scan separation and channel selections in parallel'''
self.outputms = "parallel3.mms"
mstransform(vis=self.vis, outputvis=self.outputms, spw='0:0~10,1:60~63',createmms=True,
separationaxis='scan')
self.assertTrue(os.path.exists(self.outputms))
# It should create 2 subMS, with spw=0~1
# spw=0 has 11 channels, spw=1 has 4 channels
ret = th.verifyMS(self.outputms, 2, 11, 0, ignoreflags=True)
self.assertTrue(ret[0],ret[1])
ret = th.verifyMS(self.outputms, 2, 4, 1, ignoreflags=True)
self.assertTrue(ret[0],ret[1])
def test_channels2(self):
'''partition: create MMS with spw/scan separation and channel selections'''
partition(vis=self.msfile, outputvis=self.mmsfile, spw='0:0~10,1:60~63',createmms=True,
separationaxis='auto', disableparallel=True, flagbackup=False, datacolumn='data')
self.assertTrue(os.path.exists(self.mmsfile))
# It should create 4 subMS, with spw=0~1
# spw=0 has 11 channels, spw=1 has 4 channels
ret = th.verifyMS(self.mmsfile, 2, 11, 0, ignoreflags=True)
self.assertTrue(ret[0],ret[1])
ret = th.verifyMS(self.mmsfile, 2, 4, 1, ignoreflags=True)
self.assertTrue(ret[0],ret[1])
def test_mms_float(self):
'''partition: '''
partition(vis=self.msfile,
outputvis=self.mmsfile,
spw='1,3,5',
datacolumn='FLOAT_DATA',
disableparallel=True,
flagbackup=False)
ret = th.verifyMS(self.mmsfile, 3, 512, 0, [], ignoreflags=True)
self.assertTrue(ret[0], ret[1])
ret = th.verifyMS(self.mmsfile, 3, 512, 1, [], ignoreflags=True)
self.assertTrue(ret[0], ret[1])
ret = th.verifyMS(self.mmsfile, 3, 1024, 2, [], ignoreflags=True)
self.assertTrue(ret[0], ret[1])
def test_regrid1_3(self):
'''mstransform: Default regridms with spw selection using input MMS'''
# same as test_regrid1_1
mmsfile = 'testmms13.mms'
# Create input MMS
mstransform(vis=self.vis, outputvis=mmsfile, createmms=True, disableparallel=True,
separationaxis='scan')
self.outputms = "reg13.ms"
mstransform(vis=mmsfile, outputvis=self.outputms, regridms=True, spw='1,3,5,7',
datacolumn='DATA')
self.assertTrue(os.path.exists(self.outputms))
# The regriding should be the same as the input
for i in range(4):
ret = th.verifyMS(self.outputms, 4, 64, i)
self.assertTrue(ret[0],ret[1])
listobs(self.outputms)
# Verify that some sub-tables are properly re-indexed.
spw_col = th.getVarCol(self.outputms+'/DATA_DESCRIPTION', 'SPECTRAL_WINDOW_ID')
self.assertEqual(spw_col.keys().__len__(), 4, 'Wrong number of rows in DD table')
self.assertEqual(spw_col['r1'][0], 0,'Error re-indexing DATA_DESCRIPTION table')
self.assertEqual(spw_col['r2'][0], 1,'Error re-indexing DATA_DESCRIPTION table')
self.assertEqual(spw_col['r3'][0], 2,'Error re-indexing DATA_DESCRIPTION table')
self.assertEqual(spw_col['r4'][0], 3,'Error re-indexing DATA_DESCRIPTION table')
def test_channels_mms3(self):
'''mstransform: create MMS with scan separation and channel selections'''
self.outputms = "testmms3.mms"
mstransform(vis=self.vis, outputvis=self.outputms, spw='0:0~10,1:60~63',createmms=True,
separationaxis='scan', disableparallel=True)
self.assertTrue(os.path.exists(self.outputms))
# It should create 2 subMS, with spw=0~1
# spw=0 has 11 channels, spw=1 has 4 channels
ret = th.verifyMS(self.outputms, 2, 11, 0, ignoreflags=True)
self.assertTrue(ret[0],ret[1])
ret = th.verifyMS(self.outputms, 2, 4, 1, ignoreflags=True)
self.assertTrue(ret[0],ret[1])
# The separation axis should be written to the output MMS
sepaxis = ph.axisType(self.outputms)
self.assertEqual(sepaxis, 'scan', 'AxisType is not correctly written to output MMS')
def test_split_float(self):
"""partition: split an MS with FLOAT_DATA"""
partition(
vis=self.msfile,
outputvis=self.mmsfile,
spw="1,3,5",
createmms=False,
datacolumn="FLOAT_DATA",
disableparallel=True,
flagbackup=False,
)
ret = th.verifyMS(self.mmsfile, 3, 512, 0, [], ignoreflags=True)
self.assertTrue(ret[0], ret[1])
ret = th.verifyMS(self.mmsfile, 3, 512, 1, [], ignoreflags=True)
self.assertTrue(ret[0], ret[1])
ret = th.verifyMS(self.mmsfile, 3, 1024, 2, [], ignoreflags=True)
self.assertTrue(ret[0], ret[1])
def test_channels_mms4(self):
'''mstransform: verify spw sub-table consolidation in sequential'''
self.outputms = "testmms4.mms"
mstransform(vis=self.vis, outputvis=self.outputms, spw='3,5:10~20,7,11,13',createmms=True,
separationaxis='spw', disableparallel=True)
self.assertTrue(os.path.exists(self.outputms))
# spw=5 should be spw=1 after consolidation, with 10 channels
ret = th.verifyMS(self.outputms, 7, 10, 1, ignoreflags=True)
def test_freqavg6(self):
'''mstranform: Average all channels of one spw, save as an MMS'''
# same as test_freqavg3
self.outputms = "favg6.ms"
mstransform(vis=self.vis, outputvis=self.outputms, spw='23', chanaverage=True, chanbin=128,
createmms=True, disableparallel=True)
self.assertTrue(os.path.exists(self.outputms))
ret = th.verifyMS(self.outputms, 1, 1, 0)
self.assertTrue(ret[0],ret[1])
def test_channels2(self):
'''partition: create MMS with spw/scan separation and channel selections'''
partition(vis=self.msfile,
outputvis=self.mmsfile,
spw='0:0~10,1:60~63',
createmms=True,
separationaxis='auto',
disableparallel=True,
flagbackup=False,
datacolumn='data')
self.assertTrue(os.path.exists(self.mmsfile))
# It should create 4 subMS, with spw=0~1
# spw=0 has 11 channels, spw=1 has 4 channels
ret = th.verifyMS(self.mmsfile, 2, 11, 0, ignoreflags=True)
self.assertTrue(ret[0], ret[1])
ret = th.verifyMS(self.mmsfile, 2, 4, 1, ignoreflags=True)
self.assertTrue(ret[0], ret[1])
def test_parallel2(self):
'''mstransform: create MMS with spw/scan separation and channel selections in parallel'''
self.outputms = "parallel2.mms"
mstransform(vis=self.vis, outputvis=self.outputms, spw='0:0~10,1:60~63',createmms=True,
separationaxis='auto')
self.assertTrue(os.path.exists(self.outputms))
# It should create 4 subMS, with spw=0~1
# spw=0 has 11 channels, spw=1 has 4 channels
ret = th.verifyMS(self.outputms, 2, 11, 0, ignoreflags=True)
self.assertTrue(ret[0],ret[1])
ret = th.verifyMS(self.outputms, 2, 4, 1, ignoreflags=True)
self.assertTrue(ret[0],ret[1])
# Check the FEED table
out_feed_spw = th.getVarCol(self.outputms+'/FEED', 'SPECTRAL_WINDOW_ID')
self.assertEqual(len(out_feed_spw.keys()), 8)
self.assertEqual(out_feed_spw['r1'], 0)
self.assertEqual(out_feed_spw['r5'], 1)
def test_channels1(self):
'''partition: create MMS with spw separation and channel selections'''
partition(vis=self.msfile, outputvis=self.mmsfile, spw='0~4,5:1~10',createmms=True,
separationaxis='spw', disableparallel=True, flagbackup=False)
self.assertTrue(os.path.exists(self.mmsfile))
# It should create 6 subMS, with spw=0~5
# spw=5 should have only 10 channels
ret = th.verifyMS(self.mmsfile, 6, 10, 5,ignoreflags=True)
self.assertTrue(ret[0],ret[1])
def test_freqavg8(self):
'''mstranform: Average using different bins for several spws, output MMS'''
# same as test_freqavg4
self.outputms = "favg8.ms"
mstransform(vis=self.vis, outputvis=self.outputms, spw='10,12,20', chanaverage=True,
chanbin=[128,4,10], createmms=True, separationaxis='spw',numsubms=2,
disableparallel=True)
self.assertTrue(os.path.exists(self.outputms))
# Output should be:
# spw=0 1 channel
# spw=1 32 channels
# spw=3 13 channels
ret = th.verifyMS(self.outputms, 3, 1, 0, ignoreflags=True)
self.assertTrue(ret[0],ret[1])
ret = th.verifyMS(self.outputms, 3, 32, 1, ignoreflags=True)
self.assertTrue(ret[0],ret[1])
ret = th.verifyMS(self.outputms, 3, 12, 2, ignoreflags=True)
self.assertTrue(ret[0],ret[1])
def test_parallel1(self):
'''mstransform: create MMS with spw separation and channel selections in parallel'''
self.outputms = "parallel1.mms"
mstransform(vis=self.vis, outputvis=self.outputms, spw='0~4,5:1~10',createmms=True,
separationaxis='spw')
self.assertTrue(os.path.exists(self.outputms))
# It should create 6 subMS, with spw=0~5
# spw=5 should have only 10 channels
ret = th.verifyMS(self.outputms, 6, 10, 5,ignoreflags=True)
self.assertTrue(ret[0],ret[1])
def test_monolithic_combspw1_1(self):
'''mstransform: Combine four spws into one using a monolithic-MMS'''
self.createMMS(self.vis, axis='spw',spws='0~3')
self.outputms = "monocombspw11.ms"
mstransform(vis=self.testmms, outputvis=self.outputms, datacolumn='data',combinespws=True, spw='0~3')
self.assertTrue(ParallelDataHelper.isParallelMS(self.outputms),'Output should be an MMS')
ret = th.verifyMS(self.outputms, 1, 256, 0)
self.assertTrue(ret[0],ret[1])
listobs(self.outputms, listfile='list2.obs')
self.assertTrue(os.path.exists('list2.obs'), 'Probable error in sub-table re-indexing')
def test_combspw1_4(self):
'''mstransform: Combine some channels of two spws using MMS input'''
# same test as test_combspw1_2
mmsfile = "inpmms14.mms"
# First create an MMS
mstransform(vis=self.vis, outputvis=mmsfile, spw='0,1', createmms=True)
# Now do the same as in test_combspw1_2. Datacolumn moved to DATA
self.outputms = "combspw14.ms"
mstransform(vis=mmsfile, outputvis=self.outputms, combinespws=True, spw='0:60~63,1:60~63',
datacolumn='data', disableparallel=True)
self.assertTrue(ParallelDataHelper.isParallelMS(self.outputms), 'Output should be an MMS')
# The spws contain gaps, therefore the number of channels is bigger
ret = th.verifyMS(self.outputms, 1, 68, 0)
self.assertTrue(ret[0],ret[1])
# Compare with cvel results
default(cvel)
cvel(vis=self.vis, outputvis='combcvel14.ms', spw='0:60~63,1:60~63')
ret = th.verifyMS('combcvel14.ms', 1, 68, 0)
self.assertTrue(ret[0],ret[1])
def test_channels3(self):
'''partition: verify spw sub-table consolidation'''
partition(vis=self.msfile, outputvis=self.mmsfile, spw='3,5:10~19,7,9,11,13,15',
createmms=True,separationaxis='spw', flagbackup=False, datacolumn='data')
self.assertTrue(os.path.exists(self.mmsfile))
# spw=5 should be spw=1 after consolidation, with 10 channels
ret = th.verifyMS(self.mmsfile, 7, 10, 1, ignoreflags=True)
self.assertTrue(ret[0],ret[1])
# The separation axis should be written to the output MMS
sepaxis = ph.axisType(self.mmsfile)
self.assertEqual(sepaxis, 'spw', 'Partition did not write AxisType correctly in MMS')
def test_channels_mms2(self):
'''mstransform: create MMS with spw/scan separation and channel selections'''
self.outputms = "testmms2.mms"
mstransform(vis=self.vis, outputvis=self.outputms, spw='0:0~10,1:60~63',createmms=True,
separationaxis='auto', disableparallel=True)
self.assertTrue(os.path.exists(self.outputms))
# It should create 4 subMS, with spw=0~1
# spw=0 has 11 channels, spw=1 has 4 channels
ret = th.verifyMS(self.outputms, 2, 11, 0, ignoreflags=True)
self.assertTrue(ret[0],ret[1])
ret = th.verifyMS(self.outputms, 2, 4, 1, ignoreflags=True)
self.assertTrue(ret[0],ret[1])
# Verify that some sub-tables are properly re-indexed.
spw_col = th.getVarCol(self.outputms+'/DATA_DESCRIPTION', 'SPECTRAL_WINDOW_ID')
self.assertEqual(spw_col.keys().__len__(), 2, 'Wrong number of rows in DD table')
self.assertEqual(spw_col['r1'][0], 0,'Error re-indexing DATA_DESCRIPTION table')
self.assertEqual(spw_col['r2'][0], 1,'Error re-indexing DATA_DESCRIPTION table')
# The separation axis should be written to the output MMS
sepaxis = ph.axisType(self.outputms)
self.assertEqual(sepaxis, 'scan,spw', 'AxisType is not correctly written to output MMS')
def test_channels_mms1(self):
'''mstransform: create MMS with spw separation and channel selections'''
self.outputms = "testmms1.mms"
mstransform(vis=self.vis, outputvis=self.outputms, spw='0~4,5:1~10',createmms=True,
separationaxis='spw',disableparallel=True)
self.assertTrue(os.path.exists(self.outputms))
# It should create 6 subMS, with spw=0~5
# spw=5 should have only 10 channels
ret = th.verifyMS(self.outputms, 6, 10, 5,ignoreflags=True)
self.assertTrue(ret[0],ret[1])
# The separation axis should be written to the output MMS
sepaxis = ph.axisType(self.outputms)
self.assertEqual(sepaxis, 'spw', 'AxisType is not correctly written to output MMS')
def test_channels1(self):
'''partition: create MMS with spw separation and channel selections'''
partition(vis=self.msfile,
outputvis=self.mmsfile,
spw='0~4,5:1~10',
createmms=True,
separationaxis='spw',
disableparallel=True,
flagbackup=False)
self.assertTrue(os.path.exists(self.mmsfile))
# It should create 6 subMS, with spw=0~5
# spw=5 should have only 10 channels
ret = th.verifyMS(self.mmsfile, 6, 10, 5, ignoreflags=True)
self.assertTrue(ret[0], ret[1])
def test_parallel4(self):
'''mstransform: verify spw sub-table consolidation in sequential'''
self.outputms = "parallel4.mms"
mstransform(vis=self.vis, outputvis=self.outputms, spw='3,5:10~20,7,9,15',createmms=True,
separationaxis='spw', numsubms=5)
self.assertTrue(os.path.exists(self.outputms))
# spw=5 should be spw=1 after consolidation, with 10 channels
ret = th.verifyMS(self.outputms, 7, 10, 1, ignoreflags=True)
# Check the FEED table
out_feed_spw = th.getVarCol(self.outputms+'/FEED', 'SPECTRAL_WINDOW_ID')
self.assertEqual(len(out_feed_spw.keys()), 20)
self.assertEqual(out_feed_spw['r1'], 0)
self.assertEqual(out_feed_spw['r5'], 1)
self.assertEqual(out_feed_spw['r9'], 2)
self.assertEqual(out_feed_spw['r13'], 3)
self.assertEqual(out_feed_spw['r17'], 4)
def test_parallel5(self):
'''mstransform: Do not combine spws and create MMS with axis scan in parallel.'''
self.setUp_jupiter()
self.outputms = 'parallel5.mms'
mstransform(vis=self.vis, outputvis=self.outputms, combinespws=False, spw='0,1',field = '12',
datacolumn='DATA', createmms=True, separationaxis='scan',numsubms=6)
self.assertTrue(os.path.exists(self.outputms))
# Should create 6 subMSs
mslocal = mstool()
mslocal.open(thems=self.outputms)
sublist = mslocal.getreferencedtables()
mslocal.close()
self.assertEqual(sublist.__len__(), 6, 'Should have created 6 subMSs')
ret = th.verifyMS(self.outputms, 2, 1, 0)
self.assertTrue(ret[0],ret[1])
def test_MMS_as_monolithicMS(self):
'''mstransform: MMS should be processed as a monolithic MS'''
# Create an MMS in the setup. It creates self.testmms
self.createMMS(self.vis, axis='spw', spws='2,4,6')
self.outputms = 'monolithicMMS.mms'
# Treat MMS as a monolithic MS and create an output MMS with different separation axis.
mstransform(vis=self.testmms, outputvis=self.outputms, datacolumn='data', combinespws=True)
self.assertTrue(ParallelDataHelper.isParallelMS(self.outputms),'Output should be an MMS')
# The separation axis should be copied to the output MMS
in_sepaxis = ph.axisType(self.testmms)
out_sepaxis = ph.axisType(self.outputms)
self.assertNotEqual(in_sepaxis, out_sepaxis, 'AxisTypes from input and output MMS should not match')
ret = th.verifyMS(self.outputms, 1, 320, 0)
self.assertTrue(ret[0],ret[1])
listobs(self.outputms, listfile='list1.obs')
self.assertTrue(os.path.exists('list1.obs'), 'Probable error in sub-table re-indexing')
def test_channels3(self):
'''partition: verify spw sub-table consolidation'''
partition(vis=self.msfile,
outputvis=self.mmsfile,
spw='3,5:10~19,7,9,11,13,15',
createmms=True,
separationaxis='spw',
flagbackup=False,
datacolumn='data')
self.assertTrue(os.path.exists(self.mmsfile))
# spw=5 should be spw=1 after consolidation, with 10 channels
ret = th.verifyMS(self.mmsfile, 7, 10, 1, ignoreflags=True)
self.assertTrue(ret[0], ret[1])
# The separation axis should be written to the output MMS
sepaxis = ph.axisType(self.mmsfile)
self.assertEqual(sepaxis, 'spw',
'Partition did not write AxisType correctly in MMS')
