def test_nomms(self):
'''Partition: Create a normal MS with createmms=False'''
partition(vis=self.msfile, outputvis=self.mmsfile, createmms=False)
self.assertTrue(os.path.exists(self.mmsfile),
'MMS was not created for this test')
# Sort the output MSs so that they can be compared
myms = mstool()
myms.open(self.msfile)
myms.sort('ms_sorted.ms', [
'OBSERVATION_ID', 'ARRAY_ID', 'SCAN_NUMBER', 'FIELD_ID',
'DATA_DESC_ID', 'ANTENNA1', 'ANTENNA2', 'TIME'
])
myms.done()
myms.open(self.mmsfile)
myms.sort('mms_sorted.ms', [
'OBSERVATION_ID', 'ARRAY_ID', 'SCAN_NUMBER', 'FIELD_ID',
'DATA_DESC_ID', 'ANTENNA1', 'ANTENNA2', 'TIME'
])
myms.done()
# Compare both tables. Ignore the DATA column and compare it in next line
self.assertTrue(
th.compTables(
'ms_sorted.ms', 'mms_sorted.ms',
['FLAG_CATEGORY', 'FLAG', 'WEIGHT_SPECTRUM', 'DATA']))
# Compare the DATA column
self.assertTrue(
th.compVarColTables('ms_sorted.ms', 'mms_sorted.ms', 'DATA'))
def test_scan_selection(self):
"""Partition: create an MMS using scan selection"""
partition(
vis=self.msfile,
outputvis=self.mmsfile,
separationaxis="scan",
scan="1,2,3,11",
flagbackup=False,
disableparallel=True,
)
self.assertTrue(os.path.exists(self.mmsfile), "MMS was not created for this test")
# Take the dictionary and compare with original MS
thisdict = listpartition(vis=self.mmsfile, createdict=True)
# Compare nrows of all scans in selection
slist = ph.getMMSScans(thisdict)
self.assertEqual(slist.__len__(), 4)
for s in slist:
mmsN = ph.getMMSScanNrows(thisdict, s)
msN = ph.getScanNrows(self.msfile, s)
self.assertEqual(mmsN, msN, "Nrows in scan=%s differs: mms_nrows=%s <--> ms_nrows=%s" % (s, mmsN, msN))
# Compare spw IDs
for s in slist:
mms_spw = ph.getSpwIds(self.mmsfile, s)
ms_spw = ph.getSpwIds(self.msfile, s)
self.assertEqual(
mms_spw, ms_spw, "list of spws in scan=%s differs: " "mms_spw=%s <--> ms_spw=%s" % (s, mmsN, msN)
)
def runPartition(self, ms, mmsdir, createlink, axis, subms):
'''Run partition with default values to create an MMS.
ms --> full pathname of the MS
mmsdir --> directory to save the MMS to
createlink --> when True, it will create a symbolic link to the
just created MMS in the same directory with extension .ms
axis --> separationaxis to use (spw, scan, auto)
subms --> number of subMss to create
'''
from tasks import partition
if not os.path.lexists(ms):
return False
# Create MMS name
bname = os.path.basename(ms)
if bname.endswith('.ms'):
mmsname = bname.replace('.ms', '.mms')
else:
mmsname = bname + '.mms'
mms = os.path.join(self.mmsdir, mmsname)
if os.path.lexists(mms):
casalog.post('Output MMS already exist -->' + mms, 'ERROR')
return False
# Check for remainings of corrupted mms
corrupted = mms.replace('.mms', '.data')
if os.path.exists(corrupted):
casalog.post('Cleaning up left overs', 'WARN')
shutil.rmtree(corrupted)
# Run partition
default('partition')
partition(vis=ms,
outputvis=mms,
createmms=True,
datacolumn='all',
flagbackup=False,
separationaxis=axis,
numsubms=subms)
casalog.origin('convertToMMS')
# Check if MMS was created
if not os.path.exists(mms):
casalog.post('Cannot create MMS ->' + mms, 'ERROR')
return False
# If requested, create a link to this MMS
if createlink:
here = os.getcwd()
os.chdir(mmsdir)
mmsname = os.path.basename(mms)
lms = mmsname.replace('.mms', '.ms')
casalog.post('Creating symbolic link to MMS')
os.symlink(mmsname, lms)
os.chdir(here)
return True
def test_flagsrestore(self):
"""Partition: check that we can restore the flags"""
# Delete any flagversions
if os.path.exists(self.msfile + ".flagversions"):
shutil.rmtree(self.msfile + ".flagversions")
# Unflag the MS
flagdata(vis=self.msfile, mode="unflag", flagbackup=False)
# Run partition and create the .flagversions
partition(vis=self.msfile, outputvis=self.mmsfile, createmms=True, disableparallel=True, datacolumn="data")
self.assertTrue(os.path.exists(self.mmsfile + ".flagversions"))
# Flag spw=9 and then spw=7 in the MMS
flagdata(vis=self.mmsfile, mode="manual", spw="9", flagbackup=True)
flagdata(vis=self.mmsfile, mode="manual", spw="7", flagbackup=True)
# There should be flags in spw=7 and 9 in MMS
res = flagdata(vis=self.mmsfile, mode="summary")
self.assertEqual(res["flagged"], 549888)
# Restore the original flags (there should be none)
flagmanager(vis=self.mmsfile, mode="restore", versionname="partition_1")
res = flagdata(vis=self.mmsfile, mode="summary")
self.assertEqual(res["flagged"], 0)
def test_all_columns(self):
'''Partition: datacolumn=all'''
partition(vis=self.msfile,
outputvis=self.mmsfile,
datacolumn='all',
flagbackup=False,
disableparallel=True)
self.assertTrue(os.path.exists(self.mmsfile),
'MMS was not created for this test')
# Take the dictionary and compare with original MS
thisdict = listpartition(vis=self.mmsfile, createdict=True)
# Compare nrows of all scans in selection
slist = ph.getMMSScans(thisdict)
self.assertEqual(slist.__len__(), 2)
for s in slist:
mmsN = ph.getMMSScanNrows(thisdict, s)
msN = ph.getScanNrows(self.msfile, s)
self.assertEqual(
mmsN, msN,
'Nrows in scan=%s differs: mms_nrows=%s <--> ms_nrows=%s' %
(s, mmsN, msN))
# Compare spw IDs
for s in slist:
mms_spw = ph.getSpwIds(self.mmsfile, s)
ms_spw = ph.getSpwIds(self.msfile, s)
self.assertEqual(mms_spw, ms_spw, 'list of spws in scan=%s differs: '\
'mms_spw=%s <--> ms_spw=%s' %(s, mms_spw, ms_spw))
def test_default_sequential(self):
'''Partition: create an MMS with default values in sequential'''
partition(vis=self.msfile,
outputvis=self.mmsfile,
disableparallel=True)
self.assertTrue(os.path.exists(self.mmsfile),
'MMS was not created for this test')
# Take the dictionary and compare with original MS
thisdict = listpartition(vis=self.mmsfile, createdict=True)
# Get scans of MMS
slist = ph.getMMSScans(thisdict)
for s in slist:
mmsN = ph.getMMSScanNrows(thisdict, s)
msN = ph.getScanNrows(self.msfile, s)
self.assertEqual(
mmsN, msN,
'Nrows in scan=%s differs: mms_nrows=%s <--> ms_nrows=%s' %
(s, mmsN, msN))
# Compare spw IDs of MS and MMS
for s in slist:
mms_spw = ph.getSpwIds(self.mmsfile, s)
ms_spw = ph.getSpwIds(self.msfile, s)
self.assertEqual(mms_spw, ms_spw, 'list of spws in scan=%s differs: '\
'mms_spw=%s <--> ms_spw=%s' %(s, mmsN, msN))
# The separation axis should be written to the output MMS
sepaxis = ph.axisType(self.mmsfile)
self.assertEqual(sepaxis, 'scan,spw',
'Partition did not write AxisType correctly in MMS')
def createMMS(self,
msfile,
axis='auto',
scans='',
spws='',
fields='',
numms=64):
'''Create MMSs for tests with input MMS'''
prefix = msfile[:-len('.ms')]
# Create an MMS for the tests
self.testmms = "cvel_input_test.mms"
default(partition)
if os.path.exists(self.testmms):
os.system("rm -rf " + self.testmms)
os.system("rm -rf " + self.testmms + '.flagversions')
print "................. Creating test MMS .................."
partition(vis=msfile,
outputvis=self.testmms,
flagbackup=False,
separationaxis=axis,
scan=scans,
spw=spws,
field=fields,
numsubms=numms)
def test_spw_selection(self):
'''Partition: create an MMS separated by spws with spw=2,4 selection'''
partition(vis=self.msfile, outputvis=self.mmsfile, separationaxis='spw', spw='2,4',
flagbackup=False, disableparallel=True)
self.assertTrue(os.path.exists(self.mmsfile), 'MMS was not created for this test')
# Take the dictionary and compare with original MS
thisdict = listpartition(vis=self.mmsfile, createdict=True)
# Dictionary with selection to compare with original MS
mysel = {'spw':'2,4'}
# Compare nrows of all scans in selection
slist = ph.getMMSScans(thisdict)
for s in slist:
mmsN = ph.getMMSScanNrows(thisdict, s)
msN = ph.getScanNrows(self.msfile, s, selection=mysel)
self.assertEqual(mmsN, msN, 'Nrows in scan=%s differs: mms_nrows=%s <--> ms_nrows=%s'
%(s, mmsN, msN))
# spwids are re-indexed. The expected IDs are:
# ms_spw = 2 --> mms_spw = 0
# ms_spw = 4 --> mms_spw = 1
# Check that MMS spw IDs have been re-indexed properly
indexed_ids = range(2)
for s in slist:
mms_spw = ph.getSpwIds(self.mmsfile, s)
self.assertEqual(mms_spw, indexed_ids, 'spw IDs were not properly re-indexed')
def test_spw_separation(self):
'''Partition: create an MMS separated by spws with observation selection'''
# NOTE: ms.getscansummary() used in ph.getScanNrows does not honour several observation
# IDs, therefore I need to selection by obs id in partition
partition(vis=self.msfile, outputvis=self.mmsfile, separationaxis='spw', observation='2',
flagbackup=False, disableparallel=True)
self.assertTrue(os.path.exists(self.mmsfile), 'MMS was not created for this test')
# Take the dictionary and compare with original MS
thisdict = listpartition(vis=self.mmsfile, createdict=True)
# Dictionary with selection to compare with original MS
mysel = {'observation':'2'}
# Compare nrows of all scans in MMS and MS
slist = ph.getMMSScans(thisdict)
for s in slist:
mmsN = ph.getMMSScanNrows(thisdict, s)
msN = ph.getScanNrows(self.msfile, s, selection=mysel)
self.assertEqual(mmsN, msN, 'Nrows in scan=%s differs: mms_nrows=%s <--> ms_nrows=%s'
%(s, mmsN, msN))
# spwids are re-indexed. The expected IDs are:
# ms_spw = 2 --> mms_spw = 0
# ms_spw = 3 --> mms_spw = 1, etc.
# Check that MMS spw IDs have been re-indexed properly
indexed_ids = range(4)
for s in slist:
mms_spw = ph.getSpwIds(self.mmsfile, s)
self.assertEqual(mms_spw, indexed_ids, 'spw IDs were not properly re-indexed')
def test_sepaxis(self):
'''Partition: separationaxis=auto'''
partition(vis=self.msfile, outputvis=self.mmsfile, spw='0~11',separationaxis='auto',
flagbackup=False, disableparallel=True, datacolumn='data')
# partition(vis=self.msfile, outputvis=self.mmsfile,separationaxis='auto')
self.assertTrue(os.path.exists(self.mmsfile), 'MMS was not created for this test')
# Dictionary with selection to compare with original MS
mysel = {'spw':'0~11'}
# Take the dictionary and compare with original MS
thisdict = listpartition(vis=self.mmsfile, createdict=True)
# Compare nrows of all scans in selection
slist = ph.getMMSScans(thisdict)
self.assertEqual(slist.__len__(), 2)
for s in slist:
mmsN = ph.getMMSScanNrows(thisdict, s)
msN = ph.getScanNrows(self.msfile, s, selection=mysel)
# msN = ph.getScanNrows(self.msfile, s)
self.assertEqual(mmsN, msN, 'Nrows in scan=%s differs: mms_nrows=%s <--> ms_nrows=%s'
%(s, mmsN, msN))
# Compare spw IDs
for s in slist:
mms_spw = ph.getSpwIds(self.mmsfile, s)
ms_spw = ph.getSpwIds(self.msfile, s, selection=mysel)
# ms_spw = ph.getSpwIds(self.msfile, s)
self.assertEqual(mms_spw, ms_spw, 'list of spws in scan=%s differs: '\
'mms_spw=%s <--> ms_spw=%s' %(s, mms_spw, ms_spw))
# The separation axis should be written to the output MMS
sepaxis = ph.axisType(self.mmsfile)
self.assertEqual(sepaxis, 'scan,spw', 'Partition did not write AxisType correctly in MMS')
def test_default_sequential(self):
'''Partition: create an MMS with default values in sequential'''
partition(vis=self.msfile, outputvis=self.mmsfile, disableparallel=True)
self.assertTrue(os.path.exists(self.mmsfile), 'MMS was not created for this test')
# Take the dictionary and compare with original MS
thisdict = listpartition(vis=self.mmsfile, createdict=True)
# Get scans of MMS
slist = ph.getMMSScans(thisdict)
for s in slist:
mmsN = ph.getMMSScanNrows(thisdict, s)
msN = ph.getScanNrows(self.msfile, s)
self.assertEqual(mmsN, msN, 'Nrows in scan=%s differs: mms_nrows=%s <--> ms_nrows=%s'
%(s, mmsN, msN))
# Compare spw IDs of MS and MMS
for s in slist:
mms_spw = ph.getSpwIds(self.mmsfile, s)
ms_spw = ph.getSpwIds(self.msfile, s)
self.assertEqual(mms_spw, ms_spw, 'list of spws in scan=%s differs: '\
'mms_spw=%s <--> ms_spw=%s' %(s, mmsN, msN))
# The separation axis should be written to the output MMS
sepaxis = ph.axisType(self.mmsfile)
self.assertEqual(sepaxis, 'scan,spw', 'Partition did not write AxisType correctly in MMS')
def test_flagversions(self):
'''Partition: check that the .flagversions is created'''
# Run partition and create the .flagversions
partition(vis=self.msfile,
outputvis=self.mmsfile,
createmms=True,
disableparallel=True,
datacolumn='data')
self.assertTrue(os.path.exists(self.mmsfile + '.flagversions'))
# Check that the number of backups in MMS is correct
aflocal = aftool()
aflocal.open(self.mmsfile)
nv = aflocal.getflagversionlist()
aflocal.done()
self.assertEqual(len(nv), 3)
# Run flagdata on MMS to check if it works well.
flagdata(vis=self.mmsfile, mode='unflag', flagbackup=True)
# Check that the number of backups in MMS is correct
aflocal = aftool()
aflocal.open(self.mmsfile)
nvref = aflocal.getflagversionlist()
aflocal.done()
self.assertEqual(len(nvref), 4)
def test_scan_spw(self):
'''Partition: separationaxis=scan with spw selection'''
partition(vis=self.msfile, outputvis=self.mmsfile, separationaxis='scan',
spw='1~4,10,11', flagbackup=False, datacolumn='data')
self.assertTrue(os.path.exists(self.mmsfile), 'MMS was not created for this test')
self.assertTrue(os.path.exists(self.msfile),'Make sure the input MS is not deleted inside the task')
# Take the dictionary and compare with original MS
thisdict = listpartition(vis=self.mmsfile, createdict=True)
# Dictionary with selection to compare with original MS
mysel = {'spw':'1~4,10,11'}
# Compare nrows of all scans in selection
slist = ph.getMMSScans(thisdict)
for s in slist:
mmsN = ph.getMMSScanNrows(thisdict, s)
msN = ph.getScanNrows(self.msfile, s, selection=mysel)
self.assertEqual(mmsN, msN, 'Nrows in scan=%s differs: mms_nrows=%s <--> ms_nrows=%s'
%(s, mmsN, msN))
# The comparison should be
# ms_spw = 1 --> mms_spw = 0
# ms_spw = 2 --> mms_spw = 1, etc.
# Check that MMS spw IDs have been re-indexed properly
indexed_ids = range(6)
for s in slist:
mms_spw = ph.getSpwIds(self.mmsfile, s)
self.assertEqual(mms_spw, indexed_ids, 'spw IDs were not properly re-indexed')
# The separation axis should be written to the output MMS
sepaxis = ph.axisType(self.mmsfile)
self.assertEqual(sepaxis, 'scan', 'Partition did not write AxisType correctly in MMS')
def test_all_columns(self):
'''Partition: datacolumn=all'''
partition(vis=self.msfile, outputvis=self.mmsfile, datacolumn='all',
flagbackup=False, disableparallel=True)
self.assertTrue(os.path.exists(self.mmsfile), 'MMS was not created for this test')
# Take the dictionary and compare with original MS
thisdict = listpartition(vis=self.mmsfile, createdict=True)
# Compare nrows of all scans in selection
slist = ph.getMMSScans(thisdict)
self.assertEqual(slist.__len__(), 2)
for s in slist:
mmsN = ph.getMMSScanNrows(thisdict, s)
msN = ph.getScanNrows(self.msfile, s)
self.assertEqual(mmsN, msN, 'Nrows in scan=%s differs: mms_nrows=%s <--> ms_nrows=%s'
%(s, mmsN, msN))
# Compare spw IDs
for s in slist:
mms_spw = ph.getSpwIds(self.mmsfile, s)
ms_spw = ph.getSpwIds(self.msfile, s)
self.assertEqual(mms_spw, ms_spw, 'list of spws in scan=%s differs: '\
'mms_spw=%s <--> ms_spw=%s' %(s, mms_spw, ms_spw))
def test_flagsrestore(self):
'''Partition: check that we can restore the flags'''
# Delete any flagversions
if os.path.exists(self.msfile+'.flagversions'):
shutil.rmtree(self.msfile+'.flagversions')
# Unflag the MS
flagdata(vis=self.msfile, mode='unflag', flagbackup=False)
# Run partition and create the .flagversions
partition(vis=self.msfile, outputvis=self.mmsfile, createmms=True,
disableparallel=True, datacolumn='data')
self.assertTrue(os.path.exists(self.mmsfile+'.flagversions'))
# Flag spw=9 and then spw=7 in the MMS
flagdata(vis=self.mmsfile, mode='manual', spw='9', flagbackup=True)
flagdata(vis=self.mmsfile, mode='manual', spw='7', flagbackup=True)
# There should be flags in spw=7 and 9 in MMS
res = flagdata(vis=self.mmsfile, mode='summary')
self.assertEqual(res['flagged'],549888)
# Restore the original flags (there should be none)
flagmanager(vis=self.mmsfile, mode='restore', versionname='partition_1')
res = flagdata(vis=self.mmsfile, mode='summary')
self.assertEqual(res['flagged'],0)
def test_merge_subtables_after_partiton_alma(self):
'''mstransform: Check that sub-tables are properly merged after partiton'''
self.outputms = "parted-alma.ms"
partition(self.vis, outputvis=self.outputms,separationaxis='spw',numsubms=4,flagbackup=False)
subtable = "/FEED"
sort_order = "SPECTRAL_WINDOW_ID, ANTENNA_ID, FEED_ID, TIME"
res = th.compareSubTables(self.outputms+subtable,self.vis+subtable,sort_order)
self.assertTrue(res[0],"Error comparing " + res[1] + " column from " + subtable + " sub-table")
subtable = "/CALDEVICE"
sort_order = "SPECTRAL_WINDOW_ID, ANTENNA_ID, FEED_ID, TIME"
excluded_cols = ['CAL_LOAD_NAMES','NOISE_CAL','CAL_EFF','TEMPERATURE_LOAD']
res = th.compareSubTables(self.outputms+subtable,self.vis+subtable,sort_order,excluded_cols)
self.assertTrue(res[0],"Error comparing " + res[1] + " column from " + subtable + " sub-table")
subtable = "/SYSCAL"
sort_order = "SPECTRAL_WINDOW_ID, ANTENNA_ID, FEED_ID, TIME"
excluded_cols = ['TCAL_SPECTRUM','TRX_SPECTRUM','TSKY_SPECTRUM','TSYS_SPECTRUM','TANT_SPECTRUM','TANT_TSYS_SPECTRUM']
res = th.compareSubTables(self.outputms+subtable,self.vis+subtable,sort_order,excluded_cols)
self.assertTrue(res[0],"Error comparing " + res[1] + " column from " + subtable + " sub-table")
subtable = "/SOURCE"
sort_order = "SPECTRAL_WINDOW_ID, TIME, SOURCE_ID"
excluded_cols = ['POSITION','TRANSITION','REST_FREQUENCY','SYSVEL']
res = th.compareSubTables(self.outputms+subtable,self.vis+subtable,sort_order,excluded_cols)
self.assertTrue(res[0],"Error comparing " + res[1] + " column from " + subtable + " sub-table")
subtable = "/SPECTRAL_WINDOW"
excluded_cols = ['ASSOC_SPW_ID','ASSOC_NATURE']
res = th.compareSubTables(self.outputms+subtable,self.vis+subtable,None,excluded_cols)
self.assertTrue(res[0],"Error comparing " + res[1] + " column from " + subtable + " sub-table")
def test_flagsrestore(self):
'''Partition: check that we can restore the flags'''
# Delete any flagversions
if os.path.exists(self.msfile + '.flagversions'):
shutil.rmtree(self.msfile + '.flagversions')
# Unflag the MS
flagdata(vis=self.msfile, mode='unflag', flagbackup=False)
# Run partition and create the .flagversions
partition(vis=self.msfile,
outputvis=self.mmsfile,
createmms=True,
disableparallel=True,
datacolumn='data')
self.assertTrue(os.path.exists(self.mmsfile + '.flagversions'))
# Flag spw=9 and then spw=7 in the MMS
flagdata(vis=self.mmsfile, mode='manual', spw='9', flagbackup=True)
flagdata(vis=self.mmsfile, mode='manual', spw='7', flagbackup=True)
# There should be flags in spw=7 and 9 in MMS
res = flagdata(vis=self.mmsfile, mode='summary')
self.assertEqual(res['flagged'], 549888)
# Restore the original flags (there should be none)
flagmanager(vis=self.mmsfile,
mode='restore',
versionname='partition_1')
res = flagdata(vis=self.mmsfile, mode='summary')
self.assertEqual(res['flagged'], 0)
def test_merge_subtables_after_partiton_evla(self):
"""mstransform: Check that sub-tables are properly merged after partiton"""
self.outputms = "parted-evla.ms"
partition(self.vis, outputvis=self.outputms, separationaxis="spw", numsubms=4, flagbackup=False)
subtable = "/FEED"
sort_order = "SPECTRAL_WINDOW_ID, ANTENNA_ID, FEED_ID, TIME"
res = th.compareSubTables(self.outputms + subtable, self.vis + subtable, sort_order)
self.assertTrue(res[0], "Error comparing " + res[1] + " column from " + subtable + " sub-table")
subtable = "/CALDEVICE"
sort_order = "SPECTRAL_WINDOW_ID, ANTENNA_ID, FEED_ID, TIME"
excluded_cols = ["CAL_LOAD_NAMES", "NOISE_CAL", "CAL_EFF", "TEMPERATURE_LOAD"]
res = th.compareSubTables(self.outputms + subtable, self.vis + subtable, sort_order, excluded_cols)
self.assertTrue(res[0], "Error comparing " + res[1] + " column from " + subtable + " sub-table")
subtable = "/SYSPOWER"
sort_order = "SPECTRAL_WINDOW_ID, ANTENNA_ID, FEED_ID, TIME"
excluded_cols = ["SWITCHED_DIFF", "SWITCHED_SUM", "REQUANTIZER_GAIN", "TANT_SPECTRUM"]
res = th.compareSubTables(self.outputms + subtable, self.vis + subtable, sort_order, excluded_cols)
self.assertTrue(res[0], "Error comparing " + res[1] + " column from " + subtable + " sub-table")
subtable = "/SOURCE"
sort_order = "SPECTRAL_WINDOW_ID, TIME, SOURCE_ID"
excluded_cols = ["POSITION", "TRANSITION", "REST_FREQUENCY", "SYSVEL"]
res = th.compareSubTables(self.outputms + subtable, self.vis + subtable, sort_order, excluded_cols)
self.assertTrue(res[0], "Error comparing " + res[1] + " column from " + subtable + " sub-table")
subtable = "/SPECTRAL_WINDOW"
excluded_cols = ["ASSOC_SPW_ID", "ASSOC_NATURE"]
res = th.compareSubTables(self.outputms + subtable, self.vis + subtable, None, excluded_cols)
self.assertTrue(res[0], "Error comparing " + res[1] + " column from " + subtable + " sub-table")
def test_nomms(self):
"""Partition: Create a normal MS with createmms=False"""
partition(vis=self.msfile, outputvis=self.mmsfile, createmms=False)
self.assertTrue(os.path.exists(self.mmsfile), "MMS was not created for this test")
# Sort the output MSs so that they can be compared
myms = mstool()
myms.open(self.msfile)
myms.sort(
"ms_sorted.ms",
["OBSERVATION_ID", "ARRAY_ID", "SCAN_NUMBER", "FIELD_ID", "DATA_DESC_ID", "ANTENNA1", "ANTENNA2", "TIME"],
)
myms.done()
myms.open(self.mmsfile)
myms.sort(
"mms_sorted.ms",
["OBSERVATION_ID", "ARRAY_ID", "SCAN_NUMBER", "FIELD_ID", "DATA_DESC_ID", "ANTENNA1", "ANTENNA2", "TIME"],
)
myms.done()
# Compare both tables. Ignore the DATA column and compare it in next line
self.assertTrue(
th.compTables("ms_sorted.ms", "mms_sorted.ms", ["FLAG_CATEGORY", "FLAG", "WEIGHT_SPECTRUM", "DATA"])
)
# Compare the DATA column
self.assertTrue(th.compVarColTables("ms_sorted.ms", "mms_sorted.ms", "DATA"))
def test_partition_balanced_multiple_scan(self):
partition(self.vis, outputvis=self.outputms, separationaxis='auto', numsubms=2, disableparallel=True)
listpartition_dict = listpartition(self.outputms, createdict=True)
self.assertEqual(len(listpartition_dict), 2)
self.assertEqual(len(listpartition_dict[0]['scanId']), 26)
self.assertEqual(len(listpartition_dict[1]['scanId']), 26)
self.assertEqual(sum([x['nrows'] for x in listpartition_dict[1]['scanId'].values()]), 4344)
self.assertEqual(sum([x['nrows'] for x in listpartition_dict[0]['scanId'].values()]), 4344)
def test_baseline_max(self):
'''partition: create an MMS per baseline axis. Use the maximum number of baselines'''
self.outputms = 'baseline_max.mms'
partition(self.vis, outputvis=self.outputms, separationaxis='baseline', numsubms=50, flagbackup=False)
# Take the dictionary and compare with original MS
thisdict = listpartition(vis=self.outputms, createdict=True)
self.assertEqual(len(thisdict.keys()), 6, 'There should be 6 subMSs in output MMS')
def test_baseline1(self):
'''partition: create an MMS per baseline axis. Use the number of MPI servers'''
self.outputms = 'baseline1.mms'
partition(self.vis, outputvis=self.outputms, separationaxis='baseline', flagbackup=False)
# The separation axis should be written to the output MMS
sepaxis = ph.axisType(self.outputms)
self.assertEqual(sepaxis, 'baseline', 'Partition did not write AxisType correctly in MMS')
def test_baseline_2subms(self):
'''partition: create an MMS per baseline axis. Create 3 SubMS'''
self.outputms = 'baseline_3subms.mms'
partition(self.vis, outputvis=self.outputms, separationaxis='baseline', numsubms=3, flagbackup=False)
# Take the dictionary and compare with original MS
thisdict = listpartition(vis=self.outputms, createdict=True)
self.assertEqual(len(thisdict.keys()), 3, 'There should be 3 subMSs in output MMS')
def test_partition_balanced_repeated_nsubms_greater_than_available_pairs(self):
'''mstransform: Check that balanced mode properly handles the case when the number of requests
subMSs is greater than the number of available ddi,scan pairs'''
self.outputms = "test_partition_3c84scan1.mms"
partition(self.vis, outputvis=self.outputms,separationaxis='auto',numsubms=64,flagbackup=False)
listpartition_dict = listpartition(self.outputms, createdict=True)
self.assertEqual(len(listpartition_dict), 4, "Number of subMS should be 3")
def test_default_scan(self):
'''Partition: create an MMS with default values and axis=scan'''
partition(vis=self.msfile,
outputvis=self.mmsfile,
separationaxis='scan')
self.assertTrue(os.path.exists(self.mmsfile),
'MMS was not created for this test')
# Take the dictionary and compare with original MS
thisdict = listpartition(vis=self.mmsfile, createdict=True)
# Compare nrows of all scans
slist = ph.getMMSScans(thisdict)
for s in slist:
mmsN = ph.getMMSScanNrows(thisdict, s)
msN = ph.getScanNrows(self.msfile, s)
self.assertEqual(
mmsN, msN,
'Nrows in scan=%s differs: mms_nrows=%s <--> ms_nrows=%s' %
(s, mmsN, msN))
# Compare spw IDs
for s in slist:
mms_spw = ph.getSpwIds(self.mmsfile, s)
ms_spw = ph.getSpwIds(self.msfile, s)
self.assertEqual(mms_spw, ms_spw, 'list of spws in scan=%s differs: '\
'mms_spw=%s <--> ms_spw=%s' %(s, mmsN, msN))
# Sort the output MSs so that they can be compared
myms = mstool()
myms.open(self.msfile)
myms.sort('ms_sorted.ms', [
'OBSERVATION_ID', 'ARRAY_ID', 'SCAN_NUMBER', 'FIELD_ID',
'DATA_DESC_ID', 'ANTENNA1', 'ANTENNA2', 'TIME'
])
myms.done()
myms.open(self.mmsfile)
myms.sort('mms_sorted.ms', [
'OBSERVATION_ID', 'ARRAY_ID', 'SCAN_NUMBER', 'FIELD_ID',
'DATA_DESC_ID', 'ANTENNA1', 'ANTENNA2', 'TIME'
])
myms.done()
self.assertTrue(
th.compTables('ms_sorted.ms', 'mms_sorted.ms', [
'FLAG', 'FLAG_CATEGORY', 'TIME_CENTROID', 'WEIGHT_SPECTRUM',
'DATA'
]))
# Compare the DATA column
self.assertTrue(
th.compVarColTables('ms_sorted.ms', 'mms_sorted.ms', 'DATA'))
def runPartition(self, ms, mmsdir, createlink, axis, subms):
'''Run partition with default values to create an MMS.
ms --> full pathname of the MS
mmsdir --> directory to save the MMS to
createlink --> when True, it will create a symbolic link to the
just created MMS in the same directory with extension .ms
axis --> separationaxis to use (spw, scan, auto)
subms --> number of subMss to create
'''
from tasks import partition
if not os.path.lexists(ms):
return False
# Create MMS name
bname = os.path.basename(ms)
if bname.endswith('.ms'):
mmsname = bname.replace('.ms','.mms')
else:
mmsname = bname+'.mms'
mms = os.path.join(self.mmsdir, mmsname)
if os.path.lexists(mms):
casalog.post('Output MMS already exist -->'+mms,'ERROR')
return False
# Check for remainings of corrupted mms
corrupted = mms.replace('.mms','.data')
if os.path.exists(corrupted):
casalog.post('Cleaning up left overs','WARN')
shutil.rmtree(corrupted)
# Run partition
default('partition')
partition(vis=ms, outputvis=mms, createmms=True, datacolumn='all', flagbackup=False,
separationaxis=axis, numsubms=subms)
casalog.origin('convertToMMS')
# Check if MMS was created
if not os.path.exists(mms):
casalog.post('Cannot create MMS ->'+mms, 'ERROR')
return False
# If requested, create a link to this MMS
if createlink:
here = os.getcwd()
os.chdir(mmsdir)
mmsname = os.path.basename(mms)
lms = mmsname.replace('.mms', '.ms')
casalog.post('Creating symbolic link to MMS')
os.symlink(mmsname, lms)
os.chdir(here)
return True
def test_channels1(self):
'''partition: create MMS with spw separation and channel selections'''
partition(vis=self.msfile, outputvis=self.mmsfile, spw='0~4,5:1~10',createmms=True,
separationaxis='spw', disableparallel=True, flagbackup=False)
self.assertTrue(os.path.exists(self.mmsfile))
# It should create 6 subMS, with spw=0~5
# spw=5 should have only 10 channels
ret = th.verifyMS(self.mmsfile, 6, 10, 5,ignoreflags=True)
self.assertTrue(ret[0],ret[1])
def test_mms_float(self):
'''partition: '''
partition(vis=self.msfile, outputvis=self.mmsfile,spw='1,3,5',
datacolumn='FLOAT_DATA', disableparallel=True,flagbackup=False)
ret = th.verifyMS(self.mmsfile, 3, 512, 0, [], ignoreflags=True)
self.assertTrue(ret[0],ret[1])
ret = th.verifyMS(self.mmsfile, 3, 512, 1, [], ignoreflags=True)
self.assertTrue(ret[0],ret[1])
ret = th.verifyMS(self.mmsfile, 3, 1024, 2, [], ignoreflags=True)
self.assertTrue(ret[0],ret[1])
def test_partition_balanced_repeated_spws(self):
'''mstransform: Check that balanced mode properly distributes the chunks when one SPW is
pointed by more than one DDI (with different correlation setups)'''
self.outputms = "test_partition_balanced_repeated_spws.mms"
partition(self.vis, outputvis=self.outputms,separationaxis='auto',spw='0,2',numsubms=3,flagbackup=False)
listpartition_dict = listpartition(self.outputms, createdict=True)
self.assertEqual(listpartition_dict[0]['scanId'][1]['spwIds'], [1], "Wrong SPWIds in subMS 0")
self.assertEqual(listpartition_dict[1]['scanId'][1]['spwIds'], [0], "Wrong SPWIds in subMS 1")
self.assertEqual(listpartition_dict[2]['scanId'][1]['spwIds'], [0], "Wrong SPWIds in subMS 2")
def test_baseline1(self):
'''partition: create an MMS per baseline axis. Use the number of MPI servers'''
self.outputms = 'baseline1.mms'
partition(self.vis,
outputvis=self.outputms,
separationaxis='baseline',
flagbackup=False)
# The separation axis should be written to the output MMS
sepaxis = ph.axisType(self.outputms)
self.assertEqual(sepaxis, 'baseline',
'Partition did not write AxisType correctly in MMS')
def test_baseline_max(self):
'''partition: create an MMS per baseline axis. Use the maximum number of baselines'''
self.outputms = 'baseline_max.mms'
partition(self.vis,
outputvis=self.outputms,
separationaxis='baseline',
numsubms=50,
flagbackup=False)
# Take the dictionary and compare with original MS
thisdict = listpartition(vis=self.outputms, createdict=True)
self.assertEqual(len(thisdict.keys()), 6,
'There should be 6 subMSs in output MMS')
def test_baseline_autocorr(self):
'''partition: create an MMS per baseline axis only for auto-correlations'''
self.outputms = 'baseline_autocorr.mms'
partition(self.vis, outputvis=self.outputms, separationaxis='baseline', antenna='*&&&', flagbackup=False)
md = msmdtool()
md.open(self.outputms)
bsl = md.baselines()
md.close()
#diagnoals give the auto-corrs
ac = bsl.diagonal()
self.assertTrue(ac.all(), 'Not all auto-correlations are there')
def test_channels2(self):
'''partition: create MMS with spw/scan separation and channel selections'''
partition(vis=self.msfile, outputvis=self.mmsfile, spw='0:0~10,1:60~63',createmms=True,
separationaxis='auto', disableparallel=True, flagbackup=False, datacolumn='data')
self.assertTrue(os.path.exists(self.mmsfile))
# It should create 4 subMS, with spw=0~1
# spw=0 has 11 channels, spw=1 has 4 channels
ret = th.verifyMS(self.mmsfile, 2, 11, 0, ignoreflags=True)
self.assertTrue(ret[0],ret[1])
ret = th.verifyMS(self.mmsfile, 2, 4, 1, ignoreflags=True)
self.assertTrue(ret[0],ret[1])
def test_baseline_2subms(self):
'''partition: create an MMS per baseline axis. Create 3 SubMS'''
self.outputms = 'baseline_3subms.mms'
partition(self.vis,
outputvis=self.outputms,
separationaxis='baseline',
numsubms=3,
flagbackup=False)
# Take the dictionary and compare with original MS
thisdict = listpartition(vis=self.outputms, createdict=True)
self.assertEqual(len(thisdict.keys()), 3,
'There should be 3 subMSs in output MMS')
def test_default_scan(self):
"""Partition: create an MMS with default values and axis=scan"""
partition(vis=self.msfile, outputvis=self.mmsfile, separationaxis="scan")
self.assertTrue(os.path.exists(self.mmsfile), "MMS was not created for this test")
# Take the dictionary and compare with original MS
thisdict = listpartition(vis=self.mmsfile, createdict=True)
# Compare nrows of all scans
slist = ph.getMMSScans(thisdict)
for s in slist:
mmsN = ph.getMMSScanNrows(thisdict, s)
msN = ph.getScanNrows(self.msfile, s)
self.assertEqual(mmsN, msN, "Nrows in scan=%s differs: mms_nrows=%s <--> ms_nrows=%s" % (s, mmsN, msN))
# Compare spw IDs
for s in slist:
mms_spw = ph.getSpwIds(self.mmsfile, s)
ms_spw = ph.getSpwIds(self.msfile, s)
self.assertEqual(
mms_spw, ms_spw, "list of spws in scan=%s differs: " "mms_spw=%s <--> ms_spw=%s" % (s, mmsN, msN)
)
# Sort the output MSs so that they can be compared
myms = mstool()
myms.open(self.msfile)
myms.sort(
"ms_sorted.ms",
["OBSERVATION_ID", "ARRAY_ID", "SCAN_NUMBER", "FIELD_ID", "DATA_DESC_ID", "ANTENNA1", "ANTENNA2", "TIME"],
)
myms.done()
myms.open(self.mmsfile)
myms.sort(
"mms_sorted.ms",
["OBSERVATION_ID", "ARRAY_ID", "SCAN_NUMBER", "FIELD_ID", "DATA_DESC_ID", "ANTENNA1", "ANTENNA2", "TIME"],
)
myms.done()
self.assertTrue(
th.compTables(
"ms_sorted.ms", "mms_sorted.ms", ["FLAG", "FLAG_CATEGORY", "TIME_CENTROID", "WEIGHT_SPECTRUM", "DATA"]
)
)
# Compare the DATA column
self.assertTrue(th.compVarColTables("ms_sorted.ms", "mms_sorted.ms", "DATA"))
def test_linked_cols(self):
'''partition: Verify that SYSPOWER, POINTING and SYSCAL are links'''
self.outputms = "linkcols.mms"
partition(self.vis, outputvis=self.outputms, numsubms=4, flagbackup=False)
self.assertTrue(os.path.exists(self.outputms),'Output MMS does not exist')
# Columns SYSPOWER, POINTING and SYSCAL should be links in most subMS
mslocal = mstool()
mslocal.open(self.outputms)
subms = mslocal.getreferencedtables()
mslocal.close()
cols = ['POINTING','SYSCAL','SYSPOWER']
for col in cols:
self.assertTrue(os.path.islink(subms[1] + '/' + col))
def test_channels3(self):
'''partition: verify spw sub-table consolidation'''
partition(vis=self.msfile, outputvis=self.mmsfile, spw='3,5:10~19,7,9,11,13,15',
createmms=True,separationaxis='spw', flagbackup=False, datacolumn='data')
self.assertTrue(os.path.exists(self.mmsfile))
# spw=5 should be spw=1 after consolidation, with 10 channels
ret = th.verifyMS(self.mmsfile, 7, 10, 1, ignoreflags=True)
self.assertTrue(ret[0],ret[1])
# The separation axis should be written to the output MMS
sepaxis = ph.axisType(self.mmsfile)
self.assertEqual(sepaxis, 'spw', 'Partition did not write AxisType correctly in MMS')
def createMMS(self, msfile, axis='auto',scans='',spws='', fields='', numms=64):
'''Create MMSs for tests with input MMS'''
prefix = msfile[:-len('.ms')]
# Create an MMS for the tests
self.testmms = "cvel_input_test.mms"
default(partition)
if os.path.exists(self.testmms):
os.system("rm -rf " + self.testmms)
os.system("rm -rf " + self.testmms +'.flagversions')
print "................. Creating test MMS .................."
partition(vis=msfile, outputvis=self.testmms,flagbackup=False, separationaxis=axis, scan=scans, spw=spws, field=fields,
numsubms=numms)
def test_mms_float(self):
'''partition: '''
partition(vis=self.msfile,
outputvis=self.mmsfile,
spw='1,3,5',
datacolumn='FLOAT_DATA',
disableparallel=True,
flagbackup=False)
ret = th.verifyMS(self.mmsfile, 3, 512, 0, [], ignoreflags=True)
self.assertTrue(ret[0], ret[1])
ret = th.verifyMS(self.mmsfile, 3, 512, 1, [], ignoreflags=True)
self.assertTrue(ret[0], ret[1])
ret = th.verifyMS(self.mmsfile, 3, 1024, 2, [], ignoreflags=True)
self.assertTrue(ret[0], ret[1])
def test_partition_balanced_repeated_nsubms_greater_than_available_pairs(
self):
'''mstransform: Check that balanced mode properly handles the case when the number of requests
subMSs is greater than the number of available ddi,scan pairs'''
self.outputms = "test_partition_3c84scan1.mms"
partition(self.vis,
outputvis=self.outputms,
separationaxis='auto',
numsubms=64,
flagbackup=False)
listpartition_dict = listpartition(self.outputms, createdict=True)
self.assertEqual(len(listpartition_dict), 4,
"Number of subMS should be 3")
def test_channels1(self):
'''partition: create MMS with spw separation and channel selections'''
partition(vis=self.msfile,
outputvis=self.mmsfile,
spw='0~4,5:1~10',
createmms=True,
separationaxis='spw',
disableparallel=True,
flagbackup=False)
self.assertTrue(os.path.exists(self.mmsfile))
# It should create 6 subMS, with spw=0~5
# spw=5 should have only 10 channels
ret = th.verifyMS(self.mmsfile, 6, 10, 5, ignoreflags=True)
self.assertTrue(ret[0], ret[1])
def test_partition_balanced_multiple_scan(self):
partition(self.vis,
outputvis=self.outputms,
separationaxis='auto',
numsubms=2,
disableparallel=True)
listpartition_dict = listpartition(self.outputms, createdict=True)
self.assertEqual(len(listpartition_dict), 2)
self.assertEqual(len(listpartition_dict[0]['scanId']), 26)
self.assertEqual(len(listpartition_dict[1]['scanId']), 26)
self.assertEqual(
sum([x['nrows']
for x in listpartition_dict[1]['scanId'].values()]), 4344)
self.assertEqual(
sum([x['nrows']
for x in listpartition_dict[0]['scanId'].values()]), 4344)
def createMMS(self, msfile, column='data', axis='auto',scans='',spws=''):
'''Create MMSs for tests with input MMS'''
prefix = msfile.rstrip('.ms')
if not os.path.exists(msfile):
os.system('cp -RL '+datapath + msfile +' '+ msfile)
# Create an MMS for the tests
self.testmms = prefix + ".test.mms"
default(mstransform)
if os.path.exists(self.testmms):
os.system("rm -rf " + self.testmms)
print "................. Creating test MMS .................."
partition(vis=msfile, outputvis=self.testmms, datacolumn=column,
createmms=True,separationaxis=axis, scan=scans, spw=spws)
def test_baseline_autocorr(self):
'''partition: create an MMS per baseline axis only for auto-correlations'''
self.outputms = 'baseline_autocorr.mms'
partition(self.vis,
outputvis=self.outputms,
separationaxis='baseline',
antenna='*&&&',
flagbackup=False)
md = msmdtool()
md.open(self.outputms)
bsl = md.baselines()
md.close()
#diagnoals give the auto-corrs
ac = bsl.diagonal()
self.assertTrue(ac.all(), 'Not all auto-correlations are there')
def test_scan_spw(self):
'''Partition: separationaxis=scan with spw selection'''
partition(vis=self.msfile,
outputvis=self.mmsfile,
separationaxis='scan',
spw='1~4,10,11',
flagbackup=False,
datacolumn='data')
self.assertTrue(os.path.exists(self.mmsfile),
'MMS was not created for this test')
self.assertTrue(
os.path.exists(self.msfile),
'Make sure the input MS is not deleted inside the task')
# Take the dictionary and compare with original MS
thisdict = listpartition(vis=self.mmsfile, createdict=True)
# Dictionary with selection to compare with original MS
mysel = {'spw': '1~4,10,11'}
# Compare nrows of all scans in selection
slist = ph.getMMSScans(thisdict)
for s in slist:
mmsN = ph.getMMSScanNrows(thisdict, s)
msN = ph.getScanNrows(self.msfile, s, selection=mysel)
self.assertEqual(
mmsN, msN,
'Nrows in scan=%s differs: mms_nrows=%s <--> ms_nrows=%s' %
(s, mmsN, msN))
# The comparison should be
# ms_spw = 1 --> mms_spw = 0
# ms_spw = 2 --> mms_spw = 1, etc.
# Check that MMS spw IDs have been re-indexed properly
indexed_ids = range(6)
for s in slist:
mms_spw = ph.getSpwIds(self.mmsfile, s)
self.assertEqual(mms_spw, indexed_ids,
'spw IDs were not properly re-indexed')
# The separation axis should be written to the output MMS
sepaxis = ph.axisType(self.mmsfile)
self.assertEqual(sepaxis, 'scan',
'Partition did not write AxisType correctly in MMS')
def test_linked_cols(self):
'''partition: Verify that SYSPOWER, POINTING and SYSCAL are links'''
self.outputms = "linkcols.mms"
partition(self.vis,
outputvis=self.outputms,
numsubms=4,
flagbackup=False)
self.assertTrue(os.path.exists(self.outputms),
'Output MMS does not exist')
# Columns SYSPOWER, POINTING and SYSCAL should be links in most subMS
mslocal = mstool()
mslocal.open(self.outputms)
subms = mslocal.getreferencedtables()
mslocal.close()
cols = ['POINTING', 'SYSCAL', 'SYSPOWER']
for col in cols:
self.assertTrue(os.path.islink(subms[1] + '/' + col))
def test_baseline_partition(self):
'''partition: create an MMS per baseline for an interferometry MS'''
partition(vis=self.msfile, outputvis=self.mmsfile, spw='0,1',createmms=True,
separationaxis='baseline', flagbackup=False, datacolumn='data')
listpartition(self.mmsfile)
# This MS doesn't have auto-correlations. It has 4 antennas and 6 baselines (cross)
# There will be MSSelectionNullSelection SEVERE messages in the logfile because
# it will try to create SubMSs from auto-correlations. This is not an error!
md = msmdtool()
md.open(self.mmsfile)
bsl = md.baselines()
md.close()
# diagonal give the auto-corrs
ac = bsl.diagonal()
self.assertFalse(ac.all(), 'There should be no auto-correlations in MMS')
def test_split_float(self):
"""partition: split an MS with FLOAT_DATA"""
partition(
vis=self.msfile,
outputvis=self.mmsfile,
spw="1,3,5",
createmms=False,
datacolumn="FLOAT_DATA",
disableparallel=True,
flagbackup=False,
)
ret = th.verifyMS(self.mmsfile, 3, 512, 0, [], ignoreflags=True)
self.assertTrue(ret[0], ret[1])
ret = th.verifyMS(self.mmsfile, 3, 512, 1, [], ignoreflags=True)
self.assertTrue(ret[0], ret[1])
ret = th.verifyMS(self.mmsfile, 3, 1024, 2, [], ignoreflags=True)
self.assertTrue(ret[0], ret[1])
def test_sepaxis(self):
'''Partition: separationaxis=auto'''
partition(vis=self.msfile,
outputvis=self.mmsfile,
spw='0~11',
separationaxis='auto',
flagbackup=False,
disableparallel=True,
datacolumn='data')
# partition(vis=self.msfile, outputvis=self.mmsfile,separationaxis='auto')
self.assertTrue(os.path.exists(self.mmsfile),
'MMS was not created for this test')
# Dictionary with selection to compare with original MS
mysel = {'spw': '0~11'}
# Take the dictionary and compare with original MS
thisdict = listpartition(vis=self.mmsfile, createdict=True)
# Compare nrows of all scans in selection
slist = ph.getMMSScans(thisdict)
self.assertEqual(slist.__len__(), 2)
for s in slist:
mmsN = ph.getMMSScanNrows(thisdict, s)
msN = ph.getScanNrows(self.msfile, s, selection=mysel)
# msN = ph.getScanNrows(self.msfile, s)
self.assertEqual(
mmsN, msN,
'Nrows in scan=%s differs: mms_nrows=%s <--> ms_nrows=%s' %
(s, mmsN, msN))
# Compare spw IDs
for s in slist:
mms_spw = ph.getSpwIds(self.mmsfile, s)
ms_spw = ph.getSpwIds(self.msfile, s, selection=mysel)
# ms_spw = ph.getSpwIds(self.msfile, s)
self.assertEqual(mms_spw, ms_spw, 'list of spws in scan=%s differs: '\
'mms_spw=%s <--> ms_spw=%s' %(s, mms_spw, ms_spw))
# The separation axis should be written to the output MMS
sepaxis = ph.axisType(self.mmsfile)
self.assertEqual(sepaxis, 'scan,spw',
'Partition did not write AxisType correctly in MMS')
def test_partition_balanced_repeated_spws(self):
'''mstransform: Check that balanced mode properly distributes the chunks when one SPW is
pointed by more than one DDI (with different correlation setups)'''
self.outputms = "test_partition_balanced_repeated_spws.mms"
partition(self.vis,
outputvis=self.outputms,
separationaxis='auto',
spw='0,2',
numsubms=3,
flagbackup=False)
listpartition_dict = listpartition(self.outputms, createdict=True)
self.assertEqual(listpartition_dict[0]['scanId'][1]['spwIds'], [1],
"Wrong SPWIds in subMS 0")
self.assertEqual(listpartition_dict[1]['scanId'][1]['spwIds'], [0],
"Wrong SPWIds in subMS 1")
self.assertEqual(listpartition_dict[2]['scanId'][1]['spwIds'], [0],
"Wrong SPWIds in subMS 2")
def test_numsubms(self):
'''Partition: small numsubms value'''
# There are 16 spws; we want only 6 sub-MSs.
partition(vis=self.msfile,
outputvis=self.mmsfile,
separationaxis='spw',
numsubms=6,
flagbackup=False,
disableparallel=True,
datacolumn='corrected')
self.assertTrue(os.path.exists(self.mmsfile),
'MMS was not created for this test')
# Take the dictionary and compare with original MS
thisdict = listpartition(vis=self.mmsfile, createdict=True)
# Check the number of sub-MSs
mmslist = []
klist = thisdict.keys()
for kk in klist:
mmslist.append(thisdict[kk]['MS'])
nsubms = mmslist.__len__()
self.assertEqual(nsubms, 6, 'There should be only 6 sub-MSs')
# Check that spw list is the same in MS and MMS
spwlist = ph.getMMSSpwIds(thisdict)
# Reference list of scans in MS
slist = ph.getScanList(self.msfile)
setlist = set([])
for s in slist:
ms_spw = ph.getSpwIds(self.msfile, s)
for a in ms_spw:
setlist.add(a)
self.assertEqual(list(setlist), spwlist)
# The separation axis should be written to the output MMS
sepaxis = ph.axisType(self.mmsfile)
self.assertEqual(sepaxis, 'spw',
'Partition did not write AxisType correctly in MMS')
def test_channels2(self):
'''partition: create MMS with spw/scan separation and channel selections'''
partition(vis=self.msfile,
outputvis=self.mmsfile,
spw='0:0~10,1:60~63',
createmms=True,
separationaxis='auto',
disableparallel=True,
flagbackup=False,
datacolumn='data')
self.assertTrue(os.path.exists(self.mmsfile))
# It should create 4 subMS, with spw=0~1
# spw=0 has 11 channels, spw=1 has 4 channels
ret = th.verifyMS(self.mmsfile, 2, 11, 0, ignoreflags=True)
self.assertTrue(ret[0], ret[1])
ret = th.verifyMS(self.mmsfile, 2, 4, 1, ignoreflags=True)
self.assertTrue(ret[0], ret[1])
def test_channels3(self):
'''partition: verify spw sub-table consolidation'''
partition(vis=self.msfile,
outputvis=self.mmsfile,
spw='3,5:10~19,7,9,11,13,15',
createmms=True,
separationaxis='spw',
flagbackup=False,
datacolumn='data')
self.assertTrue(os.path.exists(self.mmsfile))
# spw=5 should be spw=1 after consolidation, with 10 channels
ret = th.verifyMS(self.mmsfile, 7, 10, 1, ignoreflags=True)
self.assertTrue(ret[0], ret[1])
# The separation axis should be written to the output MMS
sepaxis = ph.axisType(self.mmsfile)
self.assertEqual(sepaxis, 'spw',
'Partition did not write AxisType correctly in MMS')
def createMMS(self, msfile, column='data', axis='auto', scans='', spws=''):
'''Create MMSs for tests with input MMS'''
prefix = msfile.rstrip('.ms')
if not os.path.exists(msfile):
os.system('cp -RL ' + datapath + msfile + ' ' + msfile)
# Create an MMS for the tests
self.testmms = prefix + ".test.mms"
default(mstransform)
if os.path.exists(self.testmms):
os.system("rm -rf " + self.testmms)
print "................. Creating test MMS .................."
partition(vis=msfile,
outputvis=self.testmms,
datacolumn=column,
createmms=True,
separationaxis=axis,
scan=scans,
spw=spws)
def test_spw_separation(self):
'''Partition: create an MMS separated by spws with observation selection'''
# NOTE: ms.getscansummary() used in ph.getScanNrows does not honour several observation
# IDs, therefore I need to selection by obs id in partition
partition(vis=self.msfile,
outputvis=self.mmsfile,
separationaxis='spw',
observation='2',
flagbackup=False,
disableparallel=True)
self.assertTrue(os.path.exists(self.mmsfile),
'MMS was not created for this test')
# Take the dictionary and compare with original MS
thisdict = listpartition(vis=self.mmsfile, createdict=True)
# Dictionary with selection to compare with original MS
mysel = {'observation': '2'}
# Compare nrows of all scans in MMS and MS
slist = ph.getMMSScans(thisdict)
for s in slist:
mmsN = ph.getMMSScanNrows(thisdict, s)
msN = ph.getScanNrows(self.msfile, s, selection=mysel)
self.assertEqual(
mmsN, msN,
'Nrows in scan=%s differs: mms_nrows=%s <--> ms_nrows=%s' %
(s, mmsN, msN))
# spwids are re-indexed. The expected IDs are:
# ms_spw = 2 --> mms_spw = 0
# ms_spw = 3 --> mms_spw = 1, etc.
# Check that MMS spw IDs have been re-indexed properly
indexed_ids = range(4)
for s in slist:
mms_spw = ph.getSpwIds(self.mmsfile, s)
self.assertEqual(mms_spw, indexed_ids,
'spw IDs were not properly re-indexed')
def test_spw_selection(self):
'''Partition: create an MMS separated by spws with spw=2,4 selection'''
partition(vis=self.msfile,
outputvis=self.mmsfile,
separationaxis='spw',
spw='2,4',
flagbackup=False,
disableparallel=True)
self.assertTrue(os.path.exists(self.mmsfile),
'MMS was not created for this test')
# Take the dictionary and compare with original MS
thisdict = listpartition(vis=self.mmsfile, createdict=True)
# Dictionary with selection to compare with original MS
mysel = {'spw': '2,4'}
# Compare nrows of all scans in selection
slist = ph.getMMSScans(thisdict)
for s in slist:
mmsN = ph.getMMSScanNrows(thisdict, s)
msN = ph.getScanNrows(self.msfile, s, selection=mysel)
self.assertEqual(
mmsN, msN,
'Nrows in scan=%s differs: mms_nrows=%s <--> ms_nrows=%s' %
(s, mmsN, msN))
# spwids are re-indexed. The expected IDs are:
# ms_spw = 2 --> mms_spw = 0
# ms_spw = 4 --> mms_spw = 1
# Check that MMS spw IDs have been re-indexed properly
indexed_ids = range(2)
for s in slist:
mms_spw = ph.getSpwIds(self.mmsfile, s)
self.assertEqual(mms_spw, indexed_ids,
'spw IDs were not properly re-indexed')
def test_baseline_partition(self):
'''partition: create an MMS per baseline for an interferometry MS'''
partition(vis=self.msfile,
outputvis=self.mmsfile,
spw='0,1',
createmms=True,
separationaxis='baseline',
flagbackup=False,
datacolumn='data')
listpartition(self.mmsfile)
# This MS doesn't have auto-correlations. It has 4 antennas and 6 baselines (cross)
# There will be MSSelectionNullSelection SEVERE messages in the logfile because
# it will try to create SubMSs from auto-correlations. This is not an error!
md = msmdtool()
md.open(self.mmsfile)
bsl = md.baselines()
md.close()
# diagonal give the auto-corrs
ac = bsl.diagonal()
self.assertFalse(ac.all(),
'There should be no auto-correlations in MMS')
def test_merge_subtables_after_partiton_evla(self):
'''mstransform: Check that sub-tables are properly merged after partiton'''
self.outputms = "parted-evla.ms"
partition(self.vis,
outputvis=self.outputms,
separationaxis='spw',
numsubms=4,
flagbackup=False)
subtable = "/FEED"
sort_order = "SPECTRAL_WINDOW_ID, ANTENNA_ID, FEED_ID, TIME"
res = th.compareSubTables(self.outputms + subtable,
self.vis + subtable, sort_order)
self.assertTrue(
res[0], "Error comparing " + res[1] + " column from " + subtable +
" sub-table")
subtable = "/CALDEVICE"
sort_order = "SPECTRAL_WINDOW_ID, ANTENNA_ID, FEED_ID, TIME"
excluded_cols = [
'CAL_LOAD_NAMES', 'NOISE_CAL', 'CAL_EFF', 'TEMPERATURE_LOAD'
]
res = th.compareSubTables(self.outputms + subtable,
self.vis + subtable, sort_order,
excluded_cols)
self.assertTrue(
res[0], "Error comparing " + res[1] + " column from " + subtable +
" sub-table")
subtable = "/SYSPOWER"
sort_order = "SPECTRAL_WINDOW_ID, ANTENNA_ID, FEED_ID, TIME"
excluded_cols = [
'SWITCHED_DIFF', 'SWITCHED_SUM', 'REQUANTIZER_GAIN',
'TANT_SPECTRUM'
]
res = th.compareSubTables(self.outputms + subtable,
self.vis + subtable, sort_order,
excluded_cols)
self.assertTrue(
res[0], "Error comparing " + res[1] + " column from " + subtable +
" sub-table")
subtable = "/SOURCE"
sort_order = "SPECTRAL_WINDOW_ID, TIME, SOURCE_ID"
excluded_cols = ['POSITION', 'TRANSITION', 'REST_FREQUENCY', 'SYSVEL']
res = th.compareSubTables(self.outputms + subtable,
self.vis + subtable, sort_order,
excluded_cols)
self.assertTrue(
res[0], "Error comparing " + res[1] + " column from " + subtable +
" sub-table")
subtable = "/SPECTRAL_WINDOW"
excluded_cols = ['ASSOC_SPW_ID', 'ASSOC_NATURE']
res = th.compareSubTables(self.outputms + subtable,
self.vis + subtable, None, excluded_cols)
self.assertTrue(
res[0], "Error comparing " + res[1] + " column from " + subtable +
" sub-table")
def main(thislist, axis='auto', numsubms=4):
if thislist == []:
print 'Need list of tasks to run.'
usage()
os._exit(0)
print "Will create MMS for the following tasks %s" % thislist
print
# Loop through task list
for t in thislist:
if t not in TASKLIST:
print 'ERROR: task ' + t + ' is not in TASKLIST. Run this script with -l for the full list.'
os._exit(0)
# if t == 'flagdata':
# axis='scan'
mmstest(t, axis, numsubms)
from tasks import partition, importuvfits
# if 'listvis' in thislist:
# # NOTE for test_listvis data:
# # You need to run partition by hand to create an MMS for the single-dish data set
# SDPATH = DATAPATH + 'unittest/listvis/'
# SDMMS = './unittest_mms/listvis/'
#
# partition(vis=SDPATH+'OrionS_rawACSmod', outputvis=SDMMS+'OrionS_rawACSmod.mms',
# datacolumn='float_data', createmms=True, flagbackup=False)
if 'split' in thislist:
# some additional MMSs
SPLITMMSPATH = './unittest_mms/split/'
specialcase = [
'0420+417/0420+417.ms', 'viewertest/ctb80-vsm.ms',
'split/labelled_by_time+ichan.ms'
]
for myms in specialcase:
shutil.rmtree(SPLITMMSPATH + os.path.basename(myms),
ignore_errors=True)
partition(vis=DATAPATH + myms,
outputvis=SPLITMMSPATH + os.path.basename(myms),
datacolumn='all',
flagbackup=False)
# workaround for a partition shortcoming: column keywords not copied
tb.open(SPLITMMSPATH + 'hasfc.mms/SUBMSS/hasfc.0000.ms/',
nomodify=False)
tb.putcolkeyword('FLAG_CATEGORY', 'CATEGORY',
['FLAG_CMD', 'ORIGINAL', 'USER'])
tb.close()
if 'wvrgcal' in thislist:
WVRGCALMMSPATH = './unittest_mms/wvrgcal/'
WVRGCALPATH = DATAPATH + 'unittest/wvrgcal/input/'
origwd = os.getcwd()
os.chdir(WVRGCALMMSPATH)
shutil.rmtree('input', ignore_errors=True)
os.mkdir('input')
os.chdir('input')
mydirs = os.listdir(WVRGCALPATH)
for d in mydirs:
print d
if os.path.splitext(d)[1] == '.ms':
partition(vis=WVRGCALPATH + d,
outputvis=d,
datacolumn='all',
numsubms=5,
flagbackup=False)
else:
os.symlink(WVRGCALPATH + d, d)
os.chdir(origwd)
if ('concat' in thislist):
CONCATMMSPATH = './unittest_mms/concat/'
CONCATPATH = DATAPATH + 'unittest/concat/input/'
origwd = os.getcwd()
os.chdir(CONCATMMSPATH)
shutil.rmtree('input', ignore_errors=True)
os.mkdir('input')
os.chdir('input')
mydirs = os.listdir(CONCATPATH)
for d in mydirs:
print d
if os.path.splitext(d)[1] == '.ms':
partition(vis=CONCATPATH + d,
outputvis=d,
datacolumn='all',
numsubms=6,
flagbackup=False)
else:
os.symlink(CONCATPATH + d, d)
os.chdir(origwd)
if ('cvel' in thislist):
CVELPATH = DATAPATH + 'ngc4826/fitsfiles/'
MMSPATH = './unittest_mms/cvel/'
mmsdir = MMSPATH + 'ngc4826.mms'
tempdir = 'makemmsdirtemp'
os.system('mkdir ' + tempdir)
importuvfits(fitsfile=CVELPATH + 'ngc4826.ll.fits5',
vis=tempdir + '/ngc4826.ms')
partition(vis=tempdir + '/ngc4826.ms',
outputvis=MMSPATH + 'ngc4826.mms',
separationaxis='scan',
flagbackup=False,
datacolumn='all')
os.system('rm -rf ' + tempdir)
CVELPATH = DATAPATH + 'cvel/input/'
cvelfiles = [
'jupiter6cm.demo-thinned.ms',
'g19_d2usb_targets_line-shortened-thinned.ms',
'evla-highres-sample-thinned.ms'
]
MMSPATH = './unittest_mms/cvel/'
thisdir = os.getcwd()
for cvelms in cvelfiles:
mmsname = cvelms.replace('.ms', '.mms')
partition(vis=CVELPATH + cvelms,
outputvis=MMSPATH + mmsname,
separationaxis='scan',
flagbackup=False,
datacolumn='all',
numsubms=4)
os.chdir(MMSPATH)
os.system('ln -s ' + mmsname + ' ' + cvelms)
os.chdir(thisdir)
# Create the jup.mms file
mmsname = 'jup.mms'
output = MMSPATH + mmsname
split2(vis=MMSPATH + '/jupiter6cm.demo-thinned.mms',
outputvis=output,
field='JUPITER',
datacolumn='data')
tblocal = tbtool()
tblocal.open(output, nomodify=False)
a = tblocal.getcol('TIME')
delta = (54709. * 86400 - a[0])
a = a + delta
strt = a[0]
tblocal.putcol('TIME', a)
a = tblocal.getcol('TIME_CENTROID')
a = a + delta
tblocal.putcol('TIME_CENTROID', a)
tblocal.close()
tblocal.open(output + '/OBSERVATION', nomodify=False)
a = tblocal.getcol('TIME_RANGE')
delta = strt - a[0][0]
a = a + delta
tblocal.putcol('TIME_RANGE', a)
tblocal.close()
tblocal.open(output + '/FIELD', nomodify=False)
a = tblocal.getcol('TIME')
delta = strt - a[0]
a = a + delta
tblocal.putcol('TIME', a)
tblocal.close()
mslocal = mstool()
mslocal.open(output, nomodify=False)
mslocal.addephemeris(
0,
os.environ.get('CASAPATH').split()[0] +
'/data/ephemerides/JPL-Horizons/Jupiter_54708-55437dUTC.tab',
'Jupiter_54708-55437dUTC', 0)
mslocal.close()
CVELMS = DATAPATH + 'fits-import-export/input/test.ms'
MMSPATH = './unittest_mms/cvel/'
thisdir = os.getcwd()
partition(vis=CVELMS,
outputvis=MMSPATH + 'test.mms',
separationaxis='scan',
flagbackup=False,
datacolumn='all',
numsubms=4)
os.chdir(MMSPATH)
os.system('ln -s test.mms test.ms')
os.chdir(thisdir)
if ('fixvis' in thislist):
MSPATH = os.environ.get(
'CASAPATH').split()[0] + '/data/regression/0420+417/'
MSNAME = MSPATH + '0420+417.ms'
MMSPATH = './unittest_mms/fixvis/'
MMSNAME = MMSPATH + '0420+417.mms'
partition(vis=MSNAME,
outputvis=MMSNAME,
datacolumn='all',
separationaxis=axis,
numsubms=numsubms,
flagbackup=False)
# Create symlink
thisdir = os.getcwd()
os.chdir(MMSPATH)
os.system('ln -s 0420+417.mms 0420+417.ms')
os.chdir(thisdir)
def test_default(self):
'''Partition: create an MMS with default values in parallel'''
# First split off one scan to run the test faster
split(vis=self.msfile,
outputvis='split30.ms',
datacolumn='DATA',
scan='30')
msfile = 'split30.ms'
partition(vis=msfile, outputvis=self.mmsfile)
self.assertTrue(os.path.exists(self.mmsfile),
'MMS was not created for this test')
# Gather several metadata information
# for the MS
mdlocal1 = msmdtool()
mdlocal1.open(msfile)
ms_rows = mdlocal1.nrows()
ms_nscans = mdlocal1.nscans()
ms_nspws = mdlocal1.nspw()
ms_scans = mdlocal1.scannumbers()
mdlocal1.close()
# for the MMS
mdlocal2 = msmdtool()
mdlocal2.open(self.mmsfile)
mms_rows = mdlocal2.nrows()
mms_nscans = mdlocal2.nscans()
mms_nspws = mdlocal2.nspw()
mms_scans = mdlocal2.scannumbers()
mdlocal2.close()
# Compare the number of rows
self.assertEqual(ms_rows, mms_rows,
'Compare total number of rows in MS and MMS')
self.assertEqual(ms_nscans, mms_nscans, 'Compare number of scans')
self.assertEqual(ms_nspws, mms_nspws, 'Compare number of spws')
# Compare the scans
self.assertEqual(ms_scans.all(), mms_scans.all(),
'Compare all scan IDs')
try:
mdlocal1.open(msfile)
mdlocal2.open(self.mmsfile)
# Compare the spws
for i in ms_scans:
msi = mdlocal1.spwsforscan(i)
mmsi = mdlocal2.spwsforscan(i)
self.assertEqual(msi.all(), mmsi.all(),
'Compare spw Ids for a scan')
finally:
mdlocal1.close()
mdlocal2.close()
# Sort the output MSs so that they can be compared
myms = mstool()
myms.open(msfile)
myms.sort('ms_sorted.ms', [
'OBSERVATION_ID', 'ARRAY_ID', 'SCAN_NUMBER', 'FIELD_ID',
'DATA_DESC_ID', 'ANTENNA1', 'ANTENNA2', 'TIME'
])
myms.done()
myms.open(self.mmsfile)
myms.sort('mms_sorted.ms', [
'OBSERVATION_ID', 'ARRAY_ID', 'SCAN_NUMBER', 'FIELD_ID',
'DATA_DESC_ID', 'ANTENNA1', 'ANTENNA2', 'TIME'
])
myms.done()
# Ignore WEIGHT_SPECTRUM and SIGMA_SPECTRUM, which are empty columns
self.assertTrue(
th.compTables('ms_sorted.ms', 'mms_sorted.ms', [
'FLAG', 'FLAG_CATEGORY', 'TIME_CENTROID', 'WEIGHT_SPECTRUM',
'SIGMA_SPECTRUM', 'DATA'
]))
# Compare the DATA column
self.assertTrue(
th.compVarColTables('ms_sorted.ms', 'mms_sorted.ms', 'DATA'))
# The separation axis should be written to the output MMS
sepaxis = ph.axisType(self.mmsfile)
self.assertEqual(sepaxis, 'scan,spw',
'Partition did not write AxisType correctly in MMS')