def testmix4(self):
'''listpartition MMS mix4: check the sizes of the sub-MSs'''
output = 'listpartitionmix2.txt'
if os.path.exists(output):
os.system('rm -rf '+output)
listpartition(vis=self.vis, listfile=output)
self.assertTrue(os.path.exists(output))
# Compare the sizes of the sub-MSs with the output of du -hs
ff = open(output,'r')
mslist = ff.readlines()
i = 0
for l in mslist:
if i == 0:
i += 1
continue
ll = l.rstrip()
rear = ll.rpartition(' ')
front = ll.partition(' ')
# Now get the du -hs for the same sub-MS
# Step into the data directory
dusize = ph.getDiskUsage(self.visdata+front[0])
# Compare both
self.assertEqual(dusize, rear[2], '%s is not equal to %s for %s'%(dusize,rear[2],front[0]))
ff.close()
def test_mms_XXYY_selection(self):
'''mstransform: correlation='RR,LL' should select and re-index properly'''
self.outputms = '3cRRLL.mms'
# spw 0 should not be processed. The selection should happen before the MMS work
mstransform(vis=self.vis, outputvis=self.outputms, datacolumn='data', correlation='RR,LL',
createmms=True, separationaxis='auto')
msmdt = msmdtool()
msmdt.open(self.outputms)
out_dds = msmdt.datadescids()
msmdt.done()
ref = [0,1]
for i in out_dds:
self.assertEqual(out_dds[i], ref[i])
pol_col = th.getVarCol(self.outputms+'/POLARIZATION','NUM_CORR')
self.assertEqual(pol_col['r1'][0], 0,'Error in NUM_CORR of POLARIZATION table')
self.assertEqual(pol_col['r2'][0], 0,'Error in NUM_CORR of POLARIZATION table')
self.assertEqual(pol_col['r3'][0], 2,'Error in NUM_CORR of POLARIZATION table')
self.assertEqual(pol_col['r4'][0], 2,'Error in NUM_CORR of POLARIZATION table')
# Verify that POLARIZATION table is not re-sized.
corr_col = th.getVarCol(self.outputms+'/POLARIZATION', 'NUM_CORR')
self.assertEqual(corr_col.keys().__len__(), 4, 'Wrong number of rows in POLARIZATION table')
# Check the FEED table
# out_feed_spw = th.getVarCol(self.outputms+'/FEED', 'SPECTRAL_WINDOW_ID')
# self.assertEqual(len(out_feed_spw.keys()), 52)
# listobs, listpartition should not fail
listobs(self.outputms, listfile='3c_1.obs')
self.assertTrue(os.path.exists('3c_1.obs'), 'Probable error in sub-table re-indexing')
listpartition(self.outputms, listfile='3c_2.obs')
self.assertTrue(os.path.exists('3c_2.obs'), 'Probable error in sub-table re-indexing')
def testMS2(self):
'''listpartition MS2: Save to a file'''
output = 'listpartitionms.txt'
if os.path.exists(output):
os.system('rm -rf '+output)
listpartition(vis=self.vis, listfile=output)
self.assertTrue(os.path.exists(output), 'Output file does not exist')
def testmix2(self):
'''listpartition MMS mix2: Save to a file'''
output = 'listpartitionmix1.txt'
if os.path.exists(output):
os.system('rm -rf '+output)
listpartition(vis=self.vis, listfile=output)
self.assertTrue(os.path.exists(output), 'Output file %s does not exist'%output)
# Check the number of lines in the output file
ff = open(output,'r')
nlines = len(ff.readlines())
ff.close()
self.assertEqual(nlines, 33, 'Wrong number of lines in output')
def test_sepaxis(self):
'''Partition: separationaxis=auto'''
partition(vis=self.msfile, outputvis=self.mmsfile, spw='0~11',separationaxis='auto',
flagbackup=False, disableparallel=True, datacolumn='data')
# partition(vis=self.msfile, outputvis=self.mmsfile,separationaxis='auto')
self.assertTrue(os.path.exists(self.mmsfile), 'MMS was not created for this test')
# Dictionary with selection to compare with original MS
mysel = {'spw':'0~11'}
# Take the dictionary and compare with original MS
thisdict = listpartition(vis=self.mmsfile, createdict=True)
# Compare nrows of all scans in selection
slist = ph.getMMSScans(thisdict)
self.assertEqual(slist.__len__(), 2)
for s in slist:
mmsN = ph.getMMSScanNrows(thisdict, s)
msN = ph.getScanNrows(self.msfile, s, selection=mysel)
# msN = ph.getScanNrows(self.msfile, s)
self.assertEqual(mmsN, msN, 'Nrows in scan=%s differs: mms_nrows=%s <--> ms_nrows=%s'
%(s, mmsN, msN))
# Compare spw IDs
for s in slist:
mms_spw = ph.getSpwIds(self.mmsfile, s)
ms_spw = ph.getSpwIds(self.msfile, s, selection=mysel)
# ms_spw = ph.getSpwIds(self.msfile, s)
self.assertEqual(mms_spw, ms_spw, 'list of spws in scan=%s differs: '\
'mms_spw=%s <--> ms_spw=%s' %(s, mms_spw, ms_spw))
# The separation axis should be written to the output MMS
sepaxis = ph.axisType(self.mmsfile)
self.assertEqual(sepaxis, 'scan,spw', 'Partition did not write AxisType correctly in MMS')
def test_spw_selection(self):
'''Partition: create an MMS separated by spws with spw=2,4 selection'''
partition(vis=self.msfile, outputvis=self.mmsfile, separationaxis='spw', spw='2,4',
flagbackup=False, disableparallel=True)
self.assertTrue(os.path.exists(self.mmsfile), 'MMS was not created for this test')
# Take the dictionary and compare with original MS
thisdict = listpartition(vis=self.mmsfile, createdict=True)
# Dictionary with selection to compare with original MS
mysel = {'spw':'2,4'}
# Compare nrows of all scans in selection
slist = ph.getMMSScans(thisdict)
for s in slist:
mmsN = ph.getMMSScanNrows(thisdict, s)
msN = ph.getScanNrows(self.msfile, s, selection=mysel)
self.assertEqual(mmsN, msN, 'Nrows in scan=%s differs: mms_nrows=%s <--> ms_nrows=%s'
%(s, mmsN, msN))
# spwids are re-indexed. The expected IDs are:
# ms_spw = 2 --> mms_spw = 0
# ms_spw = 4 --> mms_spw = 1
# Check that MMS spw IDs have been re-indexed properly
indexed_ids = range(2)
for s in slist:
mms_spw = ph.getSpwIds(self.mmsfile, s)
self.assertEqual(mms_spw, indexed_ids, 'spw IDs were not properly re-indexed')
def test_scan_spw(self):
'''Partition: separationaxis=scan with spw selection'''
partition(vis=self.msfile, outputvis=self.mmsfile, separationaxis='scan',
spw='1~4,10,11', flagbackup=False, datacolumn='data')
self.assertTrue(os.path.exists(self.mmsfile), 'MMS was not created for this test')
self.assertTrue(os.path.exists(self.msfile),'Make sure the input MS is not deleted inside the task')
# Take the dictionary and compare with original MS
thisdict = listpartition(vis=self.mmsfile, createdict=True)
# Dictionary with selection to compare with original MS
mysel = {'spw':'1~4,10,11'}
# Compare nrows of all scans in selection
slist = ph.getMMSScans(thisdict)
for s in slist:
mmsN = ph.getMMSScanNrows(thisdict, s)
msN = ph.getScanNrows(self.msfile, s, selection=mysel)
self.assertEqual(mmsN, msN, 'Nrows in scan=%s differs: mms_nrows=%s <--> ms_nrows=%s'
%(s, mmsN, msN))
# The comparison should be
# ms_spw = 1 --> mms_spw = 0
# ms_spw = 2 --> mms_spw = 1, etc.
# Check that MMS spw IDs have been re-indexed properly
indexed_ids = range(6)
for s in slist:
mms_spw = ph.getSpwIds(self.mmsfile, s)
self.assertEqual(mms_spw, indexed_ids, 'spw IDs were not properly re-indexed')
# The separation axis should be written to the output MMS
sepaxis = ph.axisType(self.mmsfile)
self.assertEqual(sepaxis, 'scan', 'Partition did not write AxisType correctly in MMS')
def test_scan_selection(self):
"""Partition: create an MMS using scan selection"""
partition(
vis=self.msfile,
outputvis=self.mmsfile,
separationaxis="scan",
scan="1,2,3,11",
flagbackup=False,
disableparallel=True,
)
self.assertTrue(os.path.exists(self.mmsfile), "MMS was not created for this test")
# Take the dictionary and compare with original MS
thisdict = listpartition(vis=self.mmsfile, createdict=True)
# Compare nrows of all scans in selection
slist = ph.getMMSScans(thisdict)
self.assertEqual(slist.__len__(), 4)
for s in slist:
mmsN = ph.getMMSScanNrows(thisdict, s)
msN = ph.getScanNrows(self.msfile, s)
self.assertEqual(mmsN, msN, "Nrows in scan=%s differs: mms_nrows=%s <--> ms_nrows=%s" % (s, mmsN, msN))
# Compare spw IDs
for s in slist:
mms_spw = ph.getSpwIds(self.mmsfile, s)
ms_spw = ph.getSpwIds(self.msfile, s)
self.assertEqual(
mms_spw, ms_spw, "list of spws in scan=%s differs: " "mms_spw=%s <--> ms_spw=%s" % (s, mmsN, msN)
)
def test_default_sequential(self):
'''Partition: create an MMS with default values in sequential'''
partition(vis=self.msfile,
outputvis=self.mmsfile,
disableparallel=True)
self.assertTrue(os.path.exists(self.mmsfile),
'MMS was not created for this test')
# Take the dictionary and compare with original MS
thisdict = listpartition(vis=self.mmsfile, createdict=True)
# Get scans of MMS
slist = ph.getMMSScans(thisdict)
for s in slist:
mmsN = ph.getMMSScanNrows(thisdict, s)
msN = ph.getScanNrows(self.msfile, s)
self.assertEqual(
mmsN, msN,
'Nrows in scan=%s differs: mms_nrows=%s <--> ms_nrows=%s' %
(s, mmsN, msN))
# Compare spw IDs of MS and MMS
for s in slist:
mms_spw = ph.getSpwIds(self.mmsfile, s)
ms_spw = ph.getSpwIds(self.msfile, s)
self.assertEqual(mms_spw, ms_spw, 'list of spws in scan=%s differs: '\
'mms_spw=%s <--> ms_spw=%s' %(s, mmsN, msN))
# The separation axis should be written to the output MMS
sepaxis = ph.axisType(self.mmsfile)
self.assertEqual(sepaxis, 'scan,spw',
'Partition did not write AxisType correctly in MMS')
def test_all_columns(self):
'''Partition: datacolumn=all'''
partition(vis=self.msfile, outputvis=self.mmsfile, datacolumn='all',
flagbackup=False, disableparallel=True)
self.assertTrue(os.path.exists(self.mmsfile), 'MMS was not created for this test')
# Take the dictionary and compare with original MS
thisdict = listpartition(vis=self.mmsfile, createdict=True)
# Compare nrows of all scans in selection
slist = ph.getMMSScans(thisdict)
self.assertEqual(slist.__len__(), 2)
for s in slist:
mmsN = ph.getMMSScanNrows(thisdict, s)
msN = ph.getScanNrows(self.msfile, s)
self.assertEqual(mmsN, msN, 'Nrows in scan=%s differs: mms_nrows=%s <--> ms_nrows=%s'
%(s, mmsN, msN))
# Compare spw IDs
for s in slist:
mms_spw = ph.getSpwIds(self.mmsfile, s)
ms_spw = ph.getSpwIds(self.msfile, s)
self.assertEqual(mms_spw, ms_spw, 'list of spws in scan=%s differs: '\
'mms_spw=%s <--> ms_spw=%s' %(s, mms_spw, ms_spw))
def test_default_sequential(self):
'''Partition: create an MMS with default values in sequential'''
partition(vis=self.msfile, outputvis=self.mmsfile, disableparallel=True)
self.assertTrue(os.path.exists(self.mmsfile), 'MMS was not created for this test')
# Take the dictionary and compare with original MS
thisdict = listpartition(vis=self.mmsfile, createdict=True)
# Get scans of MMS
slist = ph.getMMSScans(thisdict)
for s in slist:
mmsN = ph.getMMSScanNrows(thisdict, s)
msN = ph.getScanNrows(self.msfile, s)
self.assertEqual(mmsN, msN, 'Nrows in scan=%s differs: mms_nrows=%s <--> ms_nrows=%s'
%(s, mmsN, msN))
# Compare spw IDs of MS and MMS
for s in slist:
mms_spw = ph.getSpwIds(self.mmsfile, s)
ms_spw = ph.getSpwIds(self.msfile, s)
self.assertEqual(mms_spw, ms_spw, 'list of spws in scan=%s differs: '\
'mms_spw=%s <--> ms_spw=%s' %(s, mmsN, msN))
# The separation axis should be written to the output MMS
sepaxis = ph.axisType(self.mmsfile)
self.assertEqual(sepaxis, 'scan,spw', 'Partition did not write AxisType correctly in MMS')
def test_all_columns(self):
'''Partition: datacolumn=all'''
partition(vis=self.msfile,
outputvis=self.mmsfile,
datacolumn='all',
flagbackup=False,
disableparallel=True)
self.assertTrue(os.path.exists(self.mmsfile),
'MMS was not created for this test')
# Take the dictionary and compare with original MS
thisdict = listpartition(vis=self.mmsfile, createdict=True)
# Compare nrows of all scans in selection
slist = ph.getMMSScans(thisdict)
self.assertEqual(slist.__len__(), 2)
for s in slist:
mmsN = ph.getMMSScanNrows(thisdict, s)
msN = ph.getScanNrows(self.msfile, s)
self.assertEqual(
mmsN, msN,
'Nrows in scan=%s differs: mms_nrows=%s <--> ms_nrows=%s' %
(s, mmsN, msN))
# Compare spw IDs
for s in slist:
mms_spw = ph.getSpwIds(self.mmsfile, s)
ms_spw = ph.getSpwIds(self.msfile, s)
self.assertEqual(mms_spw, ms_spw, 'list of spws in scan=%s differs: '\
'mms_spw=%s <--> ms_spw=%s' %(s, mms_spw, ms_spw))
def test_float_data_mms_baseline_auto(self):
'''importasdm: Create an MMS with a FLOAT_DATA column separated per baseline '''
myasdmname = 'uid___A002_X6218fb_X264'
themsname = myasdmname + ".ms"
# The ocorr_mode='ao' option will create a FLOAT_DATA column instead of DATA
importasdm(myasdmname,
vis=themsname,
ocorr_mode='ao',
createmms=True,
scans='1',
separationaxis='baseline',
numsubms=4)
self.assertTrue(ParallelDataHelper.isParallelMS(themsname),
'Output is not a Multi-MS')
self.assertTrue(len(th.getColDesc(themsname, 'FLOAT_DATA')) > 0)
md = msmdtool()
md.open(themsname)
bsl = md.baselines()
md.close()
#diagnoals give the auto-corrs
ac = bsl.diagonal()
self.assertTrue(ac.all(), 'Not all auto-correlations are there')
# Take the dictionary and compare with original MS
thisdict = listpartition(vis=themsname, createdict=True)
self.assertEqual(len(thisdict.keys()), 4,
'There should be 4 subMSs in output MMS')
def test_spw_separation(self):
'''Partition: create an MMS separated by spws with observation selection'''
# NOTE: ms.getscansummary() used in ph.getScanNrows does not honour several observation
# IDs, therefore I need to selection by obs id in partition
partition(vis=self.msfile, outputvis=self.mmsfile, separationaxis='spw', observation='2',
flagbackup=False, disableparallel=True)
self.assertTrue(os.path.exists(self.mmsfile), 'MMS was not created for this test')
# Take the dictionary and compare with original MS
thisdict = listpartition(vis=self.mmsfile, createdict=True)
# Dictionary with selection to compare with original MS
mysel = {'observation':'2'}
# Compare nrows of all scans in MMS and MS
slist = ph.getMMSScans(thisdict)
for s in slist:
mmsN = ph.getMMSScanNrows(thisdict, s)
msN = ph.getScanNrows(self.msfile, s, selection=mysel)
self.assertEqual(mmsN, msN, 'Nrows in scan=%s differs: mms_nrows=%s <--> ms_nrows=%s'
%(s, mmsN, msN))
# spwids are re-indexed. The expected IDs are:
# ms_spw = 2 --> mms_spw = 0
# ms_spw = 3 --> mms_spw = 1, etc.
# Check that MMS spw IDs have been re-indexed properly
indexed_ids = range(4)
for s in slist:
mms_spw = ph.getSpwIds(self.mmsfile, s)
self.assertEqual(mms_spw, indexed_ids, 'spw IDs were not properly re-indexed')
def test_partition_balanced_multiple_scan(self):
partition(self.vis, outputvis=self.outputms, separationaxis='auto', numsubms=2, disableparallel=True)
listpartition_dict = listpartition(self.outputms, createdict=True)
self.assertEqual(len(listpartition_dict), 2)
self.assertEqual(len(listpartition_dict[0]['scanId']), 26)
self.assertEqual(len(listpartition_dict[1]['scanId']), 26)
self.assertEqual(sum([x['nrows'] for x in listpartition_dict[1]['scanId'].values()]), 4344)
self.assertEqual(sum([x['nrows'] for x in listpartition_dict[0]['scanId'].values()]), 4344)
def test_baseline_max(self):
'''partition: create an MMS per baseline axis. Use the maximum number of baselines'''
self.outputms = 'baseline_max.mms'
partition(self.vis, outputvis=self.outputms, separationaxis='baseline', numsubms=50, flagbackup=False)
# Take the dictionary and compare with original MS
thisdict = listpartition(vis=self.outputms, createdict=True)
self.assertEqual(len(thisdict.keys()), 6, 'There should be 6 subMSs in output MMS')
def test_baseline_2subms(self):
'''partition: create an MMS per baseline axis. Create 3 SubMS'''
self.outputms = 'baseline_3subms.mms'
partition(self.vis, outputvis=self.outputms, separationaxis='baseline', numsubms=3, flagbackup=False)
# Take the dictionary and compare with original MS
thisdict = listpartition(vis=self.outputms, createdict=True)
self.assertEqual(len(thisdict.keys()), 3, 'There should be 3 subMSs in output MMS')
def test_baseline_partition(self):
'''partition: create an MMS per baseline for an interferometry MS'''
partition(vis=self.msfile, outputvis=self.mmsfile, spw='0,1',createmms=True,
separationaxis='baseline', flagbackup=False, datacolumn='data')
listpartition(self.mmsfile)
# This MS doesn't have auto-correlations. It has 4 antennas and 6 baselines (cross)
# There will be MSSelectionNullSelection SEVERE messages in the logfile because
# it will try to create SubMSs from auto-correlations. This is not an error!
md = msmdtool()
md.open(self.mmsfile)
bsl = md.baselines()
md.close()
# diagonal give the auto-corrs
ac = bsl.diagonal()
self.assertFalse(ac.all(), 'There should be no auto-correlations in MMS')
def test_partition_balanced_repeated_nsubms_greater_than_available_pairs(self):
'''mstransform: Check that balanced mode properly handles the case when the number of requests
subMSs is greater than the number of available ddi,scan pairs'''
self.outputms = "test_partition_3c84scan1.mms"
partition(self.vis, outputvis=self.outputms,separationaxis='auto',numsubms=64,flagbackup=False)
listpartition_dict = listpartition(self.outputms, createdict=True)
self.assertEqual(len(listpartition_dict), 4, "Number of subMS should be 3")
def test_default_scan(self):
'''Partition: create an MMS with default values and axis=scan'''
partition(vis=self.msfile,
outputvis=self.mmsfile,
separationaxis='scan')
self.assertTrue(os.path.exists(self.mmsfile),
'MMS was not created for this test')
# Take the dictionary and compare with original MS
thisdict = listpartition(vis=self.mmsfile, createdict=True)
# Compare nrows of all scans
slist = ph.getMMSScans(thisdict)
for s in slist:
mmsN = ph.getMMSScanNrows(thisdict, s)
msN = ph.getScanNrows(self.msfile, s)
self.assertEqual(
mmsN, msN,
'Nrows in scan=%s differs: mms_nrows=%s <--> ms_nrows=%s' %
(s, mmsN, msN))
# Compare spw IDs
for s in slist:
mms_spw = ph.getSpwIds(self.mmsfile, s)
ms_spw = ph.getSpwIds(self.msfile, s)
self.assertEqual(mms_spw, ms_spw, 'list of spws in scan=%s differs: '\
'mms_spw=%s <--> ms_spw=%s' %(s, mmsN, msN))
# Sort the output MSs so that they can be compared
myms = mstool()
myms.open(self.msfile)
myms.sort('ms_sorted.ms', [
'OBSERVATION_ID', 'ARRAY_ID', 'SCAN_NUMBER', 'FIELD_ID',
'DATA_DESC_ID', 'ANTENNA1', 'ANTENNA2', 'TIME'
])
myms.done()
myms.open(self.mmsfile)
myms.sort('mms_sorted.ms', [
'OBSERVATION_ID', 'ARRAY_ID', 'SCAN_NUMBER', 'FIELD_ID',
'DATA_DESC_ID', 'ANTENNA1', 'ANTENNA2', 'TIME'
])
myms.done()
self.assertTrue(
th.compTables('ms_sorted.ms', 'mms_sorted.ms', [
'FLAG', 'FLAG_CATEGORY', 'TIME_CENTROID', 'WEIGHT_SPECTRUM',
'DATA'
]))
# Compare the DATA column
self.assertTrue(
th.compVarColTables('ms_sorted.ms', 'mms_sorted.ms', 'DATA'))
def test_partition_balanced_repeated_spws(self):
'''mstransform: Check that balanced mode properly distributes the chunks when one SPW is
pointed by more than one DDI (with different correlation setups)'''
self.outputms = "test_partition_balanced_repeated_spws.mms"
partition(self.vis, outputvis=self.outputms,separationaxis='auto',spw='0,2',numsubms=3,flagbackup=False)
listpartition_dict = listpartition(self.outputms, createdict=True)
self.assertEqual(listpartition_dict[0]['scanId'][1]['spwIds'], [1], "Wrong SPWIds in subMS 0")
self.assertEqual(listpartition_dict[1]['scanId'][1]['spwIds'], [0], "Wrong SPWIds in subMS 1")
self.assertEqual(listpartition_dict[2]['scanId'][1]['spwIds'], [0], "Wrong SPWIds in subMS 2")
def test_baseline_max(self):
'''partition: create an MMS per baseline axis. Use the maximum number of baselines'''
self.outputms = 'baseline_max.mms'
partition(self.vis,
outputvis=self.outputms,
separationaxis='baseline',
numsubms=50,
flagbackup=False)
# Take the dictionary and compare with original MS
thisdict = listpartition(vis=self.outputms, createdict=True)
self.assertEqual(len(thisdict.keys()), 6,
'There should be 6 subMSs in output MMS')
def test_baseline_2subms(self):
'''partition: create an MMS per baseline axis. Create 3 SubMS'''
self.outputms = 'baseline_3subms.mms'
partition(self.vis,
outputvis=self.outputms,
separationaxis='baseline',
numsubms=3,
flagbackup=False)
# Take the dictionary and compare with original MS
thisdict = listpartition(vis=self.outputms, createdict=True)
self.assertEqual(len(thisdict.keys()), 3,
'There should be 3 subMSs in output MMS')
def test_default_scan(self):
"""Partition: create an MMS with default values and axis=scan"""
partition(vis=self.msfile, outputvis=self.mmsfile, separationaxis="scan")
self.assertTrue(os.path.exists(self.mmsfile), "MMS was not created for this test")
# Take the dictionary and compare with original MS
thisdict = listpartition(vis=self.mmsfile, createdict=True)
# Compare nrows of all scans
slist = ph.getMMSScans(thisdict)
for s in slist:
mmsN = ph.getMMSScanNrows(thisdict, s)
msN = ph.getScanNrows(self.msfile, s)
self.assertEqual(mmsN, msN, "Nrows in scan=%s differs: mms_nrows=%s <--> ms_nrows=%s" % (s, mmsN, msN))
# Compare spw IDs
for s in slist:
mms_spw = ph.getSpwIds(self.mmsfile, s)
ms_spw = ph.getSpwIds(self.msfile, s)
self.assertEqual(
mms_spw, ms_spw, "list of spws in scan=%s differs: " "mms_spw=%s <--> ms_spw=%s" % (s, mmsN, msN)
)
# Sort the output MSs so that they can be compared
myms = mstool()
myms.open(self.msfile)
myms.sort(
"ms_sorted.ms",
["OBSERVATION_ID", "ARRAY_ID", "SCAN_NUMBER", "FIELD_ID", "DATA_DESC_ID", "ANTENNA1", "ANTENNA2", "TIME"],
)
myms.done()
myms.open(self.mmsfile)
myms.sort(
"mms_sorted.ms",
["OBSERVATION_ID", "ARRAY_ID", "SCAN_NUMBER", "FIELD_ID", "DATA_DESC_ID", "ANTENNA1", "ANTENNA2", "TIME"],
)
myms.done()
self.assertTrue(
th.compTables(
"ms_sorted.ms", "mms_sorted.ms", ["FLAG", "FLAG_CATEGORY", "TIME_CENTROID", "WEIGHT_SPECTRUM", "DATA"]
)
)
# Compare the DATA column
self.assertTrue(th.compVarColTables("ms_sorted.ms", "mms_sorted.ms", "DATA"))
def test_baseline_partition(self):
'''partition: create an MMS per baseline for an interferometry MS'''
partition(vis=self.msfile,
outputvis=self.mmsfile,
spw='0,1',
createmms=True,
separationaxis='baseline',
flagbackup=False,
datacolumn='data')
listpartition(self.mmsfile)
# This MS doesn't have auto-correlations. It has 4 antennas and 6 baselines (cross)
# There will be MSSelectionNullSelection SEVERE messages in the logfile because
# it will try to create SubMSs from auto-correlations. This is not an error!
md = msmdtool()
md.open(self.mmsfile)
bsl = md.baselines()
md.close()
# diagonal give the auto-corrs
ac = bsl.diagonal()
self.assertFalse(ac.all(),
'There should be no auto-correlations in MMS')
def test_partition_balanced_repeated_nsubms_greater_than_available_pairs(
self):
'''mstransform: Check that balanced mode properly handles the case when the number of requests
subMSs is greater than the number of available ddi,scan pairs'''
self.outputms = "test_partition_3c84scan1.mms"
partition(self.vis,
outputvis=self.outputms,
separationaxis='auto',
numsubms=64,
flagbackup=False)
listpartition_dict = listpartition(self.outputms, createdict=True)
self.assertEqual(len(listpartition_dict), 4,
"Number of subMS should be 3")
def test_partition_balanced_multiple_scan(self):
partition(self.vis,
outputvis=self.outputms,
separationaxis='auto',
numsubms=2,
disableparallel=True)
listpartition_dict = listpartition(self.outputms, createdict=True)
self.assertEqual(len(listpartition_dict), 2)
self.assertEqual(len(listpartition_dict[0]['scanId']), 26)
self.assertEqual(len(listpartition_dict[1]['scanId']), 26)
self.assertEqual(
sum([x['nrows']
for x in listpartition_dict[1]['scanId'].values()]), 4344)
self.assertEqual(
sum([x['nrows']
for x in listpartition_dict[0]['scanId'].values()]), 4344)
def testmix3(self):
'''listpartition MMS mix3: Create an output dictionary'''
resdict = listpartition(vis=self.vis, createdict=True)
nkeys = resdict.keys().__len__()
self.assertEqual(nkeys, 32)
# Check all scans in all sub-MSs
for k in resdict.keys():
subms = resdict[k]['MS']
MS = self.visdata+'/'+subms
scans = resdict[k]['scanId'].keys()
for s in scans:
nr = resdict[k]['scanId'][s]['nrows']
refN = ph.getScanNrows(MS, s)
self.assertEqual(nr, refN, '%s, scan=%s, nrows=%s do not match reference nrows=%s'\
%(MS, s, nr, refN))
def test_scan_spw(self):
'''Partition: separationaxis=scan with spw selection'''
partition(vis=self.msfile,
outputvis=self.mmsfile,
separationaxis='scan',
spw='1~4,10,11',
flagbackup=False,
datacolumn='data')
self.assertTrue(os.path.exists(self.mmsfile),
'MMS was not created for this test')
self.assertTrue(
os.path.exists(self.msfile),
'Make sure the input MS is not deleted inside the task')
# Take the dictionary and compare with original MS
thisdict = listpartition(vis=self.mmsfile, createdict=True)
# Dictionary with selection to compare with original MS
mysel = {'spw': '1~4,10,11'}
# Compare nrows of all scans in selection
slist = ph.getMMSScans(thisdict)
for s in slist:
mmsN = ph.getMMSScanNrows(thisdict, s)
msN = ph.getScanNrows(self.msfile, s, selection=mysel)
self.assertEqual(
mmsN, msN,
'Nrows in scan=%s differs: mms_nrows=%s <--> ms_nrows=%s' %
(s, mmsN, msN))
# The comparison should be
# ms_spw = 1 --> mms_spw = 0
# ms_spw = 2 --> mms_spw = 1, etc.
# Check that MMS spw IDs have been re-indexed properly
indexed_ids = range(6)
for s in slist:
mms_spw = ph.getSpwIds(self.mmsfile, s)
self.assertEqual(mms_spw, indexed_ids,
'spw IDs were not properly re-indexed')
# The separation axis should be written to the output MMS
sepaxis = ph.axisType(self.mmsfile)
self.assertEqual(sepaxis, 'scan',
'Partition did not write AxisType correctly in MMS')
def test_sepaxis(self):
'''Partition: separationaxis=auto'''
partition(vis=self.msfile,
outputvis=self.mmsfile,
spw='0~11',
separationaxis='auto',
flagbackup=False,
disableparallel=True,
datacolumn='data')
# partition(vis=self.msfile, outputvis=self.mmsfile,separationaxis='auto')
self.assertTrue(os.path.exists(self.mmsfile),
'MMS was not created for this test')
# Dictionary with selection to compare with original MS
mysel = {'spw': '0~11'}
# Take the dictionary and compare with original MS
thisdict = listpartition(vis=self.mmsfile, createdict=True)
# Compare nrows of all scans in selection
slist = ph.getMMSScans(thisdict)
self.assertEqual(slist.__len__(), 2)
for s in slist:
mmsN = ph.getMMSScanNrows(thisdict, s)
msN = ph.getScanNrows(self.msfile, s, selection=mysel)
# msN = ph.getScanNrows(self.msfile, s)
self.assertEqual(
mmsN, msN,
'Nrows in scan=%s differs: mms_nrows=%s <--> ms_nrows=%s' %
(s, mmsN, msN))
# Compare spw IDs
for s in slist:
mms_spw = ph.getSpwIds(self.mmsfile, s)
ms_spw = ph.getSpwIds(self.msfile, s, selection=mysel)
# ms_spw = ph.getSpwIds(self.msfile, s)
self.assertEqual(mms_spw, ms_spw, 'list of spws in scan=%s differs: '\
'mms_spw=%s <--> ms_spw=%s' %(s, mms_spw, ms_spw))
# The separation axis should be written to the output MMS
sepaxis = ph.axisType(self.mmsfile)
self.assertEqual(sepaxis, 'scan,spw',
'Partition did not write AxisType correctly in MMS')
def test_numsubms(self):
'''Partition: small numsubms value'''
# There are 16 spws; we want only 6 sub-MSs.
partition(vis=self.msfile,
outputvis=self.mmsfile,
separationaxis='spw',
numsubms=6,
flagbackup=False,
disableparallel=True,
datacolumn='corrected')
self.assertTrue(os.path.exists(self.mmsfile),
'MMS was not created for this test')
# Take the dictionary and compare with original MS
thisdict = listpartition(vis=self.mmsfile, createdict=True)
# Check the number of sub-MSs
mmslist = []
klist = thisdict.keys()
for kk in klist:
mmslist.append(thisdict[kk]['MS'])
nsubms = mmslist.__len__()
self.assertEqual(nsubms, 6, 'There should be only 6 sub-MSs')
# Check that spw list is the same in MS and MMS
spwlist = ph.getMMSSpwIds(thisdict)
# Reference list of scans in MS
slist = ph.getScanList(self.msfile)
setlist = set([])
for s in slist:
ms_spw = ph.getSpwIds(self.msfile, s)
for a in ms_spw:
setlist.add(a)
self.assertEqual(list(setlist), spwlist)
# The separation axis should be written to the output MMS
sepaxis = ph.axisType(self.mmsfile)
self.assertEqual(sepaxis, 'spw',
'Partition did not write AxisType correctly in MMS')
def test_numsubms(self):
"""Partition: small numsubms value"""
# There are 16 spws; we want only 6 sub-MSs.
partition(
vis=self.msfile,
outputvis=self.mmsfile,
separationaxis="spw",
numsubms=6,
flagbackup=False,
disableparallel=True,
datacolumn="corrected",
)
self.assertTrue(os.path.exists(self.mmsfile), "MMS was not created for this test")
# Take the dictionary and compare with original MS
thisdict = listpartition(vis=self.mmsfile, createdict=True)
# Check the number of sub-MSs
mmslist = []
klist = thisdict.keys()
for kk in klist:
mmslist.append(thisdict[kk]["MS"])
nsubms = mmslist.__len__()
self.assertEqual(nsubms, 6, "There should be only 6 sub-MSs")
# Check that spw list is the same in MS and MMS
spwlist = ph.getMMSSpwIds(thisdict)
# Reference list of scans in MS
slist = ph.getScanList(self.msfile)
setlist = set([])
for s in slist:
ms_spw = ph.getSpwIds(self.msfile, s)
for a in ms_spw:
setlist.add(a)
self.assertEqual(list(setlist), spwlist)
# The separation axis should be written to the output MMS
sepaxis = ph.axisType(self.mmsfile)
self.assertEqual(sepaxis, "spw", "Partition did not write AxisType correctly in MMS")
def test_partition_balanced_repeated_spws(self):
'''mstransform: Check that balanced mode properly distributes the chunks when one SPW is
pointed by more than one DDI (with different correlation setups)'''
self.outputms = "test_partition_balanced_repeated_spws.mms"
partition(self.vis,
outputvis=self.outputms,
separationaxis='auto',
spw='0,2',
numsubms=3,
flagbackup=False)
listpartition_dict = listpartition(self.outputms, createdict=True)
self.assertEqual(listpartition_dict[0]['scanId'][1]['spwIds'], [1],
"Wrong SPWIds in subMS 0")
self.assertEqual(listpartition_dict[1]['scanId'][1]['spwIds'], [0],
"Wrong SPWIds in subMS 1")
self.assertEqual(listpartition_dict[2]['scanId'][1]['spwIds'], [0],
"Wrong SPWIds in subMS 2")
def test_sepaxis(self):
"""Partition: separationaxis=auto"""
partition(
vis=self.msfile,
outputvis=self.mmsfile,
spw="0~11",
separationaxis="auto",
flagbackup=False,
disableparallel=True,
datacolumn="data",
)
# partition(vis=self.msfile, outputvis=self.mmsfile,separationaxis='auto')
self.assertTrue(os.path.exists(self.mmsfile), "MMS was not created for this test")
# Dictionary with selection to compare with original MS
mysel = {"spw": "0~11"}
# Take the dictionary and compare with original MS
thisdict = listpartition(vis=self.mmsfile, createdict=True)
# Compare nrows of all scans in selection
slist = ph.getMMSScans(thisdict)
self.assertEqual(slist.__len__(), 2)
for s in slist:
mmsN = ph.getMMSScanNrows(thisdict, s)
msN = ph.getScanNrows(self.msfile, s, selection=mysel)
# msN = ph.getScanNrows(self.msfile, s)
self.assertEqual(mmsN, msN, "Nrows in scan=%s differs: mms_nrows=%s <--> ms_nrows=%s" % (s, mmsN, msN))
# Compare spw IDs
for s in slist:
mms_spw = ph.getSpwIds(self.mmsfile, s)
ms_spw = ph.getSpwIds(self.msfile, s, selection=mysel)
# ms_spw = ph.getSpwIds(self.msfile, s)
self.assertEqual(
mms_spw, ms_spw, "list of spws in scan=%s differs: " "mms_spw=%s <--> ms_spw=%s" % (s, mms_spw, ms_spw)
)
# The separation axis should be written to the output MMS
sepaxis = ph.axisType(self.mmsfile)
self.assertEqual(sepaxis, "scan,spw", "Partition did not write AxisType correctly in MMS")
def test_default_scan(self):
'''Partition: create an MMS with default values and axis=scan'''
partition(vis=self.msfile, outputvis=self.mmsfile, separationaxis='scan')
self.assertTrue(os.path.exists(self.mmsfile), 'MMS was not created for this test')
# Take the dictionary and compare with original MS
thisdict = listpartition(vis=self.mmsfile, createdict=True)
# Compare nrows of all scans
slist = ph.getMMSScans(thisdict)
for s in slist:
mmsN = ph.getMMSScanNrows(thisdict, s)
msN = ph.getScanNrows(self.msfile, s)
self.assertEqual(mmsN, msN, 'Nrows in scan=%s differs: mms_nrows=%s <--> ms_nrows=%s'
%(s, mmsN, msN))
# Compare spw IDs
for s in slist:
mms_spw = ph.getSpwIds(self.mmsfile, s)
ms_spw = ph.getSpwIds(self.msfile, s)
self.assertEqual(mms_spw, ms_spw, 'list of spws in scan=%s differs: '\
'mms_spw=%s <--> ms_spw=%s' %(s, mmsN, msN))
# Sort the output MSs so that they can be compared
myms = mstool()
myms.open(self.msfile)
myms.sort('ms_sorted.ms',['OBSERVATION_ID','ARRAY_ID','SCAN_NUMBER','FIELD_ID','DATA_DESC_ID','ANTENNA1','ANTENNA2','TIME'])
myms.done()
myms.open(self.mmsfile)
myms.sort('mms_sorted.ms',['OBSERVATION_ID','ARRAY_ID','SCAN_NUMBER','FIELD_ID','DATA_DESC_ID','ANTENNA1','ANTENNA2','TIME'])
myms.done()
self.assertTrue(th.compTables('ms_sorted.ms', 'mms_sorted.ms',
['FLAG','FLAG_CATEGORY','TIME_CENTROID',
'WEIGHT_SPECTRUM','DATA']))
# Compare the DATA column
self.assertTrue(th.compVarColTables('ms_sorted.ms','mms_sorted.ms','DATA'))
def test_float_data_mms_baseline_auto(self):
'''importasdm: Create an MMS with a FLOAT_DATA column separated per baseline '''
myasdmname = 'uid___A002_X6218fb_X264'
themsname = myasdmname+".ms"
# The ocorr_mode='ao' option will create a FLOAT_DATA column instead of DATA
importasdm(myasdmname, vis=themsname, ocorr_mode='ao', createmms=True, scans='1', separationaxis='baseline', numsubms=4)
self.assertTrue(ParallelDataHelper.isParallelMS(themsname), 'Output is not a Multi-MS')
self.assertTrue(len(th.getColDesc(themsname, 'FLOAT_DATA')) > 0)
md = msmdtool()
md.open(themsname)
bsl = md.baselines()
md.close()
#diagnoals give the auto-corrs
ac = bsl.diagonal()
self.assertTrue(ac.all(), 'Not all auto-correlations are there')
# Take the dictionary and compare with original MS
thisdict = listpartition(vis=themsname, createdict=True)
self.assertEqual(len(thisdict.keys()), 4, 'There should be 4 subMSs in output MMS')
def test_spw_separation(self):
'''Partition: create an MMS separated by spws with observation selection'''
# NOTE: ms.getscansummary() used in ph.getScanNrows does not honour several observation
# IDs, therefore I need to selection by obs id in partition
partition(vis=self.msfile,
outputvis=self.mmsfile,
separationaxis='spw',
observation='2',
flagbackup=False,
disableparallel=True)
self.assertTrue(os.path.exists(self.mmsfile),
'MMS was not created for this test')
# Take the dictionary and compare with original MS
thisdict = listpartition(vis=self.mmsfile, createdict=True)
# Dictionary with selection to compare with original MS
mysel = {'observation': '2'}
# Compare nrows of all scans in MMS and MS
slist = ph.getMMSScans(thisdict)
for s in slist:
mmsN = ph.getMMSScanNrows(thisdict, s)
msN = ph.getScanNrows(self.msfile, s, selection=mysel)
self.assertEqual(
mmsN, msN,
'Nrows in scan=%s differs: mms_nrows=%s <--> ms_nrows=%s' %
(s, mmsN, msN))
# spwids are re-indexed. The expected IDs are:
# ms_spw = 2 --> mms_spw = 0
# ms_spw = 3 --> mms_spw = 1, etc.
# Check that MMS spw IDs have been re-indexed properly
indexed_ids = range(4)
for s in slist:
mms_spw = ph.getSpwIds(self.mmsfile, s)
self.assertEqual(mms_spw, indexed_ids,
'spw IDs were not properly re-indexed')
def test_spw_selection(self):
'''Partition: create an MMS separated by spws with spw=2,4 selection'''
partition(vis=self.msfile,
outputvis=self.mmsfile,
separationaxis='spw',
spw='2,4',
flagbackup=False,
disableparallel=True)
self.assertTrue(os.path.exists(self.mmsfile),
'MMS was not created for this test')
# Take the dictionary and compare with original MS
thisdict = listpartition(vis=self.mmsfile, createdict=True)
# Dictionary with selection to compare with original MS
mysel = {'spw': '2,4'}
# Compare nrows of all scans in selection
slist = ph.getMMSScans(thisdict)
for s in slist:
mmsN = ph.getMMSScanNrows(thisdict, s)
msN = ph.getScanNrows(self.msfile, s, selection=mysel)
self.assertEqual(
mmsN, msN,
'Nrows in scan=%s differs: mms_nrows=%s <--> ms_nrows=%s' %
(s, mmsN, msN))
# spwids are re-indexed. The expected IDs are:
# ms_spw = 2 --> mms_spw = 0
# ms_spw = 4 --> mms_spw = 1
# Check that MMS spw IDs have been re-indexed properly
indexed_ids = range(2)
for s in slist:
mms_spw = ph.getSpwIds(self.mmsfile, s)
self.assertEqual(mms_spw, indexed_ids,
'spw IDs were not properly re-indexed')
def testMS1(self):
'''listpartition MS1: Input MS'''
res = listpartition(vis=self.vis)
self.assertEqual(res, {}, "It should return an empty dictionary")
def testMS3(self):
'''listpartition MS3: Create an output dictionary'''
resdict = listpartition(vis=self.vis, createdict=True)
self.assertEqual(resdict[0]['MS'], self.vis)
