def test_split_MMS(self):
'''mstransform: Split MMS in parallel'''
# Create an MMS in the setup. It creates self.testmms
self.createMMS(self.vis, axis='scan', spws='0,1')
self.outputms = 'scan30.mms'
mstransform(vis=self.testmms, outputvis=self.outputms, datacolumn='data', scan='30')
self.assertTrue(ParallelTaskHelper.isParallelMS(self.outputms),'Output is not an MMS')
mslocal = mstool()
mslocal.open(self.outputms)
sublist = mslocal.getreferencedtables()
self.assertEqual(len(sublist), 1)
# Test DD table
msmdt = msmdtool()
msmdt.open(self.outputms)
out_dds = msmdt.datadescids()
msmdt.done()
ref = [0,1]
for i in out_dds:
self.assertEqual(out_dds[i], ref[i])
# The separation axis should be copied to the output MMS
in_sepaxis = ph.axisType(self.testmms)
out_sepaxis = ph.axisType(self.outputms)
self.assertEqual(in_sepaxis, out_sepaxis, 'AxisTypes from input and output MMS do not match')
def test_MMS1(self):
'''mstransform: input MMS should be the same as output MMS'''
# Create an MMS in the setup
self.createMMS(self.vis, axis='scan', spws='0,1')
# Create another MS and compare. They should be the same
self.outputms = 'thesame.mms'
mstransform(vis=self.testmms, outputvis=self.outputms, datacolumn='data')
self.assertTrue(ParallelDataHelper.isParallelMS(self.outputms),'Output is not an MMS')
# Sort the MSs so that they can be compared
myms = mstool()
myms.open(self.testmms)
myms.sort('input_sorted.ms',['OBSERVATION_ID','ARRAY_ID','SCAN_NUMBER','FIELD_ID','DATA_DESC_ID','ANTENNA1','ANTENNA2','TIME'])
myms.done()
myms.open(self.outputms)
myms.sort('output_sorted.ms',['OBSERVATION_ID','ARRAY_ID','SCAN_NUMBER','FIELD_ID','DATA_DESC_ID','ANTENNA1','ANTENNA2','TIME'])
myms.done()
# Compare both tables. Ignore the DATA column and compare it in next line
self.assertTrue(th.compTables('input_sorted.ms','output_sorted.ms',
['FLAG_CATEGORY','FLAG','WEIGHT_SPECTRUM','SIGMA_SPECTRUM','DATA']))
# Compare the DATA column
self.assertTrue(th.compVarColTables('input_sorted.ms','output_sorted.ms','DATA'))
# The separation axis should be copied to the output MMS
in_sepaxis = ph.axisType(self.testmms)
out_sepaxis = ph.axisType(self.outputms)
self.assertEqual(in_sepaxis, out_sepaxis, 'AxisTypes from input and output MMS do not match')
def test_split_MMS_weight_corr_sel(self):
'''mstransform: Split MMS in parallel. Check WEIGHT shape when selecting correlation'''
# Create an MMS in the setup. It creates self.testmms
self.createMMS(self.vis, axis='scan', spws='0,1')
self.outputms = 'corrRR_LL.mms'
mstransform(vis=self.testmms, outputvis=self.outputms, datacolumn='data', correlation='RR,LL',spw='0')
self.assertTrue(ParallelTaskHelper.isParallelMS(self.outputms),'Output is not an MMS')
mslocal = mstool()
mslocal.open(self.outputms)
sublist = mslocal.getreferencedtables()
self.assertEqual(len(sublist), 2)
# Test DD table
msmdt = msmdtool()
msmdt.open(self.outputms)
out_dds = msmdt.datadescids()
msmdt.done()
ref = [0]
for i in out_dds:
self.assertEqual(out_dds[i], ref[i])
# The separation axis should be copied to the output MMS
in_sepaxis = ph.axisType(self.testmms)
out_sepaxis = ph.axisType(self.outputms)
self.assertEqual(in_sepaxis, out_sepaxis, 'AxisTypes from input and output MMS do not match')
# Check the dimensions of the WEIGHT and SIGMA columns. CAS-6946
out_ws = th.getColShape(self.outputms,'WEIGHT')
out_ss = th.getColShape(self.outputms,'SIGMA')
self.assertEqual(out_ws[0],'[2]','WEIGHT shape is not correct')
self.assertEqual(out_ss[0],'[2]','SIGMA shape is not correct')
def test_default_sequential(self):
'''Partition: create an MMS with default values in sequential'''
partition(vis=self.msfile, outputvis=self.mmsfile, disableparallel=True)
self.assertTrue(os.path.exists(self.mmsfile), 'MMS was not created for this test')
# Take the dictionary and compare with original MS
thisdict = listpartition(vis=self.mmsfile, createdict=True)
# Get scans of MMS
slist = ph.getMMSScans(thisdict)
for s in slist:
mmsN = ph.getMMSScanNrows(thisdict, s)
msN = ph.getScanNrows(self.msfile, s)
self.assertEqual(mmsN, msN, 'Nrows in scan=%s differs: mms_nrows=%s <--> ms_nrows=%s'
%(s, mmsN, msN))
# Compare spw IDs of MS and MMS
for s in slist:
mms_spw = ph.getSpwIds(self.mmsfile, s)
ms_spw = ph.getSpwIds(self.msfile, s)
self.assertEqual(mms_spw, ms_spw, 'list of spws in scan=%s differs: '\
'mms_spw=%s <--> ms_spw=%s' %(s, mmsN, msN))
# The separation axis should be written to the output MMS
sepaxis = ph.axisType(self.mmsfile)
self.assertEqual(sepaxis, 'scan,spw', 'Partition did not write AxisType correctly in MMS')
def test_sepaxis(self):
'''Partition: separationaxis=auto'''
partition(vis=self.msfile, outputvis=self.mmsfile, spw='0~11',separationaxis='auto',
flagbackup=False, disableparallel=True, datacolumn='data')
# partition(vis=self.msfile, outputvis=self.mmsfile,separationaxis='auto')
self.assertTrue(os.path.exists(self.mmsfile), 'MMS was not created for this test')
# Dictionary with selection to compare with original MS
mysel = {'spw':'0~11'}
# Take the dictionary and compare with original MS
thisdict = listpartition(vis=self.mmsfile, createdict=True)
# Compare nrows of all scans in selection
slist = ph.getMMSScans(thisdict)
self.assertEqual(slist.__len__(), 2)
for s in slist:
mmsN = ph.getMMSScanNrows(thisdict, s)
msN = ph.getScanNrows(self.msfile, s, selection=mysel)
# msN = ph.getScanNrows(self.msfile, s)
self.assertEqual(mmsN, msN, 'Nrows in scan=%s differs: mms_nrows=%s <--> ms_nrows=%s'
%(s, mmsN, msN))
# Compare spw IDs
for s in slist:
mms_spw = ph.getSpwIds(self.mmsfile, s)
ms_spw = ph.getSpwIds(self.msfile, s, selection=mysel)
# ms_spw = ph.getSpwIds(self.msfile, s)
self.assertEqual(mms_spw, ms_spw, 'list of spws in scan=%s differs: '\
'mms_spw=%s <--> ms_spw=%s' %(s, mms_spw, ms_spw))
# The separation axis should be written to the output MMS
sepaxis = ph.axisType(self.mmsfile)
self.assertEqual(sepaxis, 'scan,spw', 'Partition did not write AxisType correctly in MMS')
def test_scan_spw(self):
'''Partition: separationaxis=scan with spw selection'''
partition(vis=self.msfile, outputvis=self.mmsfile, separationaxis='scan',
spw='1~4,10,11', flagbackup=False, datacolumn='data')
self.assertTrue(os.path.exists(self.mmsfile), 'MMS was not created for this test')
self.assertTrue(os.path.exists(self.msfile),'Make sure the input MS is not deleted inside the task')
# Take the dictionary and compare with original MS
thisdict = listpartition(vis=self.mmsfile, createdict=True)
# Dictionary with selection to compare with original MS
mysel = {'spw':'1~4,10,11'}
# Compare nrows of all scans in selection
slist = ph.getMMSScans(thisdict)
for s in slist:
mmsN = ph.getMMSScanNrows(thisdict, s)
msN = ph.getScanNrows(self.msfile, s, selection=mysel)
self.assertEqual(mmsN, msN, 'Nrows in scan=%s differs: mms_nrows=%s <--> ms_nrows=%s'
%(s, mmsN, msN))
# The comparison should be
# ms_spw = 1 --> mms_spw = 0
# ms_spw = 2 --> mms_spw = 1, etc.
# Check that MMS spw IDs have been re-indexed properly
indexed_ids = range(6)
for s in slist:
mms_spw = ph.getSpwIds(self.mmsfile, s)
self.assertEqual(mms_spw, indexed_ids, 'spw IDs were not properly re-indexed')
# The separation axis should be written to the output MMS
sepaxis = ph.axisType(self.mmsfile)
self.assertEqual(sepaxis, 'scan', 'Partition did not write AxisType correctly in MMS')
def test_default_sequential(self):
'''Partition: create an MMS with default values in sequential'''
partition(vis=self.msfile,
outputvis=self.mmsfile,
disableparallel=True)
self.assertTrue(os.path.exists(self.mmsfile),
'MMS was not created for this test')
# Take the dictionary and compare with original MS
thisdict = listpartition(vis=self.mmsfile, createdict=True)
# Get scans of MMS
slist = ph.getMMSScans(thisdict)
for s in slist:
mmsN = ph.getMMSScanNrows(thisdict, s)
msN = ph.getScanNrows(self.msfile, s)
self.assertEqual(
mmsN, msN,
'Nrows in scan=%s differs: mms_nrows=%s <--> ms_nrows=%s' %
(s, mmsN, msN))
# Compare spw IDs of MS and MMS
for s in slist:
mms_spw = ph.getSpwIds(self.mmsfile, s)
ms_spw = ph.getSpwIds(self.msfile, s)
self.assertEqual(mms_spw, ms_spw, 'list of spws in scan=%s differs: '\
'mms_spw=%s <--> ms_spw=%s' %(s, mmsN, msN))
# The separation axis should be written to the output MMS
sepaxis = ph.axisType(self.mmsfile)
self.assertEqual(sepaxis, 'scan,spw',
'Partition did not write AxisType correctly in MMS')
def test_baseline1(self):
'''partition: create an MMS per baseline axis. Use the number of MPI servers'''
self.outputms = 'baseline1.mms'
partition(self.vis, outputvis=self.outputms, separationaxis='baseline', flagbackup=False)
# The separation axis should be written to the output MMS
sepaxis = ph.axisType(self.outputms)
self.assertEqual(sepaxis, 'baseline', 'Partition did not write AxisType correctly in MMS')
def test_mms3(self):
'''test_mms3: Create MMS with separationaxis=spw and lazy=True'''
myasdmname = 'uid___A002_X71e4ae_X317_short'
themsname = myasdmname+".ms"
importasdm(myasdmname, createmms=True, lazy=True, scans='1,2', separationaxis='spw', flagbackup=False,
process_flags=False)
self.assertTrue(ParallelDataHelper.isParallelMS(themsname), 'Output is not a Multi-MS')
self.assertEqual(ph.axisType(themsname), 'spw', 'Separation axis of MMS should be spw')
def test_MMS_as_monolithicMS(self):
'''mstransform: MMS should be processed as a monolithic MS'''
# Create an MMS in the setup. It creates self.testmms
self.createMMS(self.vis, axis='spw', spws='2,4,6')
self.outputms = 'monolithicMMS.mms'
# Treat MMS as a monolithic MS and create an output MMS with different separation axis.
mstransform(vis=self.testmms, outputvis=self.outputms, datacolumn='data', combinespws=True)
self.assertTrue(ParallelDataHelper.isParallelMS(self.outputms),'Output should be an MMS')
# The separation axis should be copied to the output MMS
in_sepaxis = ph.axisType(self.testmms)
out_sepaxis = ph.axisType(self.outputms)
self.assertNotEqual(in_sepaxis, out_sepaxis, 'AxisTypes from input and output MMS should not match')
ret = th.verifyMS(self.outputms, 1, 320, 0)
self.assertTrue(ret[0],ret[1])
listobs(self.outputms, listfile='list1.obs')
self.assertTrue(os.path.exists('list1.obs'), 'Probable error in sub-table re-indexing')
def test_baseline1(self):
'''partition: create an MMS per baseline axis. Use the number of MPI servers'''
self.outputms = 'baseline1.mms'
partition(self.vis,
outputvis=self.outputms,
separationaxis='baseline',
flagbackup=False)
# The separation axis should be written to the output MMS
sepaxis = ph.axisType(self.outputms)
self.assertEqual(sepaxis, 'baseline',
'Partition did not write AxisType correctly in MMS')
def test_channels3(self):
'''partition: verify spw sub-table consolidation'''
partition(vis=self.msfile, outputvis=self.mmsfile, spw='3,5:10~19,7,9,11,13,15',
createmms=True,separationaxis='spw', flagbackup=False, datacolumn='data')
self.assertTrue(os.path.exists(self.mmsfile))
# spw=5 should be spw=1 after consolidation, with 10 channels
ret = th.verifyMS(self.mmsfile, 7, 10, 1, ignoreflags=True)
self.assertTrue(ret[0],ret[1])
# The separation axis should be written to the output MMS
sepaxis = ph.axisType(self.mmsfile)
self.assertEqual(sepaxis, 'spw', 'Partition did not write AxisType correctly in MMS')
def test_channels_mms1(self):
'''mstransform: create MMS with spw separation and channel selections'''
self.outputms = "testmms1.mms"
mstransform(vis=self.vis, outputvis=self.outputms, spw='0~4,5:1~10',createmms=True,
separationaxis='spw',disableparallel=True)
self.assertTrue(os.path.exists(self.outputms))
# It should create 6 subMS, with spw=0~5
# spw=5 should have only 10 channels
ret = th.verifyMS(self.outputms, 6, 10, 5,ignoreflags=True)
self.assertTrue(ret[0],ret[1])
# The separation axis should be written to the output MMS
sepaxis = ph.axisType(self.outputms)
self.assertEqual(sepaxis, 'spw', 'AxisType is not correctly written to output MMS')
def test_mms3(self):
'''test_mms3: Create MMS with separationaxis=spw and lazy=True'''
myasdmname = 'uid___A002_X71e4ae_X317_short'
themsname = myasdmname + ".ms"
importasdm(myasdmname,
createmms=True,
lazy=True,
scans='1,2',
separationaxis='spw',
flagbackup=False,
process_flags=False)
self.assertTrue(ParallelDataHelper.isParallelMS(themsname),
'Output is not a Multi-MS')
self.assertEqual(ph.axisType(themsname), 'spw',
'Separation axis of MMS should be spw')
def test_timespan_spw_axis(self):
'''mstransform: timeaverage=True, timespan=scan, separationaxis=spw'''
self.createMMS(self.vis, axis='spw',spws='1,3')
self.outputms = "spanscan_spw.mms"
mstransform(vis=self.testmms, outputvis=self.outputms, datacolumn='data',timeaverage=True,
timebin='100s',timespan='scan')
self.assertTrue(ParallelDataHelper.isParallelMS(self.outputms),'Output should be an MMS')
self.assertEqual(ph.axisType(self.outputms),'spw')
mymsmd = msmdtool()
mymsmd.open(self.outputms)
t30 = mymsmd.exposuretime(30)['value']
t31 = mymsmd.exposuretime(31)['value']
mymsmd.close()
self.assertEqual(t30, 100)
self.assertEqual(t31, 79)
def test_channels_mms3(self):
'''mstransform: create MMS with scan separation and channel selections'''
self.outputms = "testmms3.mms"
mstransform(vis=self.vis, outputvis=self.outputms, spw='0:0~10,1:60~63',createmms=True,
separationaxis='scan', disableparallel=True)
self.assertTrue(os.path.exists(self.outputms))
# It should create 2 subMS, with spw=0~1
# spw=0 has 11 channels, spw=1 has 4 channels
ret = th.verifyMS(self.outputms, 2, 11, 0, ignoreflags=True)
self.assertTrue(ret[0],ret[1])
ret = th.verifyMS(self.outputms, 2, 4, 1, ignoreflags=True)
self.assertTrue(ret[0],ret[1])
# The separation axis should be written to the output MMS
sepaxis = ph.axisType(self.outputms)
self.assertEqual(sepaxis, 'scan', 'AxisType is not correctly written to output MMS')
def test_scan_spw(self):
'''Partition: separationaxis=scan with spw selection'''
partition(vis=self.msfile,
outputvis=self.mmsfile,
separationaxis='scan',
spw='1~4,10,11',
flagbackup=False,
datacolumn='data')
self.assertTrue(os.path.exists(self.mmsfile),
'MMS was not created for this test')
self.assertTrue(
os.path.exists(self.msfile),
'Make sure the input MS is not deleted inside the task')
# Take the dictionary and compare with original MS
thisdict = listpartition(vis=self.mmsfile, createdict=True)
# Dictionary with selection to compare with original MS
mysel = {'spw': '1~4,10,11'}
# Compare nrows of all scans in selection
slist = ph.getMMSScans(thisdict)
for s in slist:
mmsN = ph.getMMSScanNrows(thisdict, s)
msN = ph.getScanNrows(self.msfile, s, selection=mysel)
self.assertEqual(
mmsN, msN,
'Nrows in scan=%s differs: mms_nrows=%s <--> ms_nrows=%s' %
(s, mmsN, msN))
# The comparison should be
# ms_spw = 1 --> mms_spw = 0
# ms_spw = 2 --> mms_spw = 1, etc.
# Check that MMS spw IDs have been re-indexed properly
indexed_ids = range(6)
for s in slist:
mms_spw = ph.getSpwIds(self.mmsfile, s)
self.assertEqual(mms_spw, indexed_ids,
'spw IDs were not properly re-indexed')
# The separation axis should be written to the output MMS
sepaxis = ph.axisType(self.mmsfile)
self.assertEqual(sepaxis, 'scan',
'Partition did not write AxisType correctly in MMS')
def test_sepaxis(self):
'''Partition: separationaxis=auto'''
partition(vis=self.msfile,
outputvis=self.mmsfile,
spw='0~11',
separationaxis='auto',
flagbackup=False,
disableparallel=True,
datacolumn='data')
# partition(vis=self.msfile, outputvis=self.mmsfile,separationaxis='auto')
self.assertTrue(os.path.exists(self.mmsfile),
'MMS was not created for this test')
# Dictionary with selection to compare with original MS
mysel = {'spw': '0~11'}
# Take the dictionary and compare with original MS
thisdict = listpartition(vis=self.mmsfile, createdict=True)
# Compare nrows of all scans in selection
slist = ph.getMMSScans(thisdict)
self.assertEqual(slist.__len__(), 2)
for s in slist:
mmsN = ph.getMMSScanNrows(thisdict, s)
msN = ph.getScanNrows(self.msfile, s, selection=mysel)
# msN = ph.getScanNrows(self.msfile, s)
self.assertEqual(
mmsN, msN,
'Nrows in scan=%s differs: mms_nrows=%s <--> ms_nrows=%s' %
(s, mmsN, msN))
# Compare spw IDs
for s in slist:
mms_spw = ph.getSpwIds(self.mmsfile, s)
ms_spw = ph.getSpwIds(self.msfile, s, selection=mysel)
# ms_spw = ph.getSpwIds(self.msfile, s)
self.assertEqual(mms_spw, ms_spw, 'list of spws in scan=%s differs: '\
'mms_spw=%s <--> ms_spw=%s' %(s, mms_spw, ms_spw))
# The separation axis should be written to the output MMS
sepaxis = ph.axisType(self.mmsfile)
self.assertEqual(sepaxis, 'scan,spw',
'Partition did not write AxisType correctly in MMS')
def test_numsubms(self):
'''Partition: small numsubms value'''
# There are 16 spws; we want only 6 sub-MSs.
partition(vis=self.msfile,
outputvis=self.mmsfile,
separationaxis='spw',
numsubms=6,
flagbackup=False,
disableparallel=True,
datacolumn='corrected')
self.assertTrue(os.path.exists(self.mmsfile),
'MMS was not created for this test')
# Take the dictionary and compare with original MS
thisdict = listpartition(vis=self.mmsfile, createdict=True)
# Check the number of sub-MSs
mmslist = []
klist = thisdict.keys()
for kk in klist:
mmslist.append(thisdict[kk]['MS'])
nsubms = mmslist.__len__()
self.assertEqual(nsubms, 6, 'There should be only 6 sub-MSs')
# Check that spw list is the same in MS and MMS
spwlist = ph.getMMSSpwIds(thisdict)
# Reference list of scans in MS
slist = ph.getScanList(self.msfile)
setlist = set([])
for s in slist:
ms_spw = ph.getSpwIds(self.msfile, s)
for a in ms_spw:
setlist.add(a)
self.assertEqual(list(setlist), spwlist)
# The separation axis should be written to the output MMS
sepaxis = ph.axisType(self.mmsfile)
self.assertEqual(sepaxis, 'spw',
'Partition did not write AxisType correctly in MMS')
def test_numsubms(self):
"""Partition: small numsubms value"""
# There are 16 spws; we want only 6 sub-MSs.
partition(
vis=self.msfile,
outputvis=self.mmsfile,
separationaxis="spw",
numsubms=6,
flagbackup=False,
disableparallel=True,
datacolumn="corrected",
)
self.assertTrue(os.path.exists(self.mmsfile), "MMS was not created for this test")
# Take the dictionary and compare with original MS
thisdict = listpartition(vis=self.mmsfile, createdict=True)
# Check the number of sub-MSs
mmslist = []
klist = thisdict.keys()
for kk in klist:
mmslist.append(thisdict[kk]["MS"])
nsubms = mmslist.__len__()
self.assertEqual(nsubms, 6, "There should be only 6 sub-MSs")
# Check that spw list is the same in MS and MMS
spwlist = ph.getMMSSpwIds(thisdict)
# Reference list of scans in MS
slist = ph.getScanList(self.msfile)
setlist = set([])
for s in slist:
ms_spw = ph.getSpwIds(self.msfile, s)
for a in ms_spw:
setlist.add(a)
self.assertEqual(list(setlist), spwlist)
# The separation axis should be written to the output MMS
sepaxis = ph.axisType(self.mmsfile)
self.assertEqual(sepaxis, "spw", "Partition did not write AxisType correctly in MMS")
def test_sepaxis(self):
"""Partition: separationaxis=auto"""
partition(
vis=self.msfile,
outputvis=self.mmsfile,
spw="0~11",
separationaxis="auto",
flagbackup=False,
disableparallel=True,
datacolumn="data",
)
# partition(vis=self.msfile, outputvis=self.mmsfile,separationaxis='auto')
self.assertTrue(os.path.exists(self.mmsfile), "MMS was not created for this test")
# Dictionary with selection to compare with original MS
mysel = {"spw": "0~11"}
# Take the dictionary and compare with original MS
thisdict = listpartition(vis=self.mmsfile, createdict=True)
# Compare nrows of all scans in selection
slist = ph.getMMSScans(thisdict)
self.assertEqual(slist.__len__(), 2)
for s in slist:
mmsN = ph.getMMSScanNrows(thisdict, s)
msN = ph.getScanNrows(self.msfile, s, selection=mysel)
# msN = ph.getScanNrows(self.msfile, s)
self.assertEqual(mmsN, msN, "Nrows in scan=%s differs: mms_nrows=%s <--> ms_nrows=%s" % (s, mmsN, msN))
# Compare spw IDs
for s in slist:
mms_spw = ph.getSpwIds(self.mmsfile, s)
ms_spw = ph.getSpwIds(self.msfile, s, selection=mysel)
# ms_spw = ph.getSpwIds(self.msfile, s)
self.assertEqual(
mms_spw, ms_spw, "list of spws in scan=%s differs: " "mms_spw=%s <--> ms_spw=%s" % (s, mms_spw, ms_spw)
)
# The separation axis should be written to the output MMS
sepaxis = ph.axisType(self.mmsfile)
self.assertEqual(sepaxis, "scan,spw", "Partition did not write AxisType correctly in MMS")
def test_channels3(self):
'''partition: verify spw sub-table consolidation'''
partition(vis=self.msfile,
outputvis=self.mmsfile,
spw='3,5:10~19,7,9,11,13,15',
createmms=True,
separationaxis='spw',
flagbackup=False,
datacolumn='data')
self.assertTrue(os.path.exists(self.mmsfile))
# spw=5 should be spw=1 after consolidation, with 10 channels
ret = th.verifyMS(self.mmsfile, 7, 10, 1, ignoreflags=True)
self.assertTrue(ret[0], ret[1])
# The separation axis should be written to the output MMS
sepaxis = ph.axisType(self.mmsfile)
self.assertEqual(sepaxis, 'spw',
'Partition did not write AxisType correctly in MMS')
def test_channels_mms2(self):
'''mstransform: create MMS with spw/scan separation and channel selections'''
self.outputms = "testmms2.mms"
mstransform(vis=self.vis, outputvis=self.outputms, spw='0:0~10,1:60~63',createmms=True,
separationaxis='auto', disableparallel=True)
self.assertTrue(os.path.exists(self.outputms))
# It should create 4 subMS, with spw=0~1
# spw=0 has 11 channels, spw=1 has 4 channels
ret = th.verifyMS(self.outputms, 2, 11, 0, ignoreflags=True)
self.assertTrue(ret[0],ret[1])
ret = th.verifyMS(self.outputms, 2, 4, 1, ignoreflags=True)
self.assertTrue(ret[0],ret[1])
# Verify that some sub-tables are properly re-indexed.
spw_col = th.getVarCol(self.outputms+'/DATA_DESCRIPTION', 'SPECTRAL_WINDOW_ID')
self.assertEqual(spw_col.keys().__len__(), 2, 'Wrong number of rows in DD table')
self.assertEqual(spw_col['r1'][0], 0,'Error re-indexing DATA_DESCRIPTION table')
self.assertEqual(spw_col['r2'][0], 1,'Error re-indexing DATA_DESCRIPTION table')
# The separation axis should be written to the output MMS
sepaxis = ph.axisType(self.outputms)
self.assertEqual(sepaxis, 'scan,spw', 'AxisType is not correctly written to output MMS')
def test_default(self):
'''Partition: create an MMS with default values in parallel'''
# First split off one scan to run the test faster
split(vis=self.msfile,
outputvis='split30.ms',
datacolumn='DATA',
scan='30')
msfile = 'split30.ms'
partition(vis=msfile, outputvis=self.mmsfile)
self.assertTrue(os.path.exists(self.mmsfile),
'MMS was not created for this test')
# Gather several metadata information
# for the MS
mdlocal1 = msmdtool()
mdlocal1.open(msfile)
ms_rows = mdlocal1.nrows()
ms_nscans = mdlocal1.nscans()
ms_nspws = mdlocal1.nspw()
ms_scans = mdlocal1.scannumbers()
mdlocal1.close()
# for the MMS
mdlocal2 = msmdtool()
mdlocal2.open(self.mmsfile)
mms_rows = mdlocal2.nrows()
mms_nscans = mdlocal2.nscans()
mms_nspws = mdlocal2.nspw()
mms_scans = mdlocal2.scannumbers()
mdlocal2.close()
# Compare the number of rows
self.assertEqual(ms_rows, mms_rows,
'Compare total number of rows in MS and MMS')
self.assertEqual(ms_nscans, mms_nscans, 'Compare number of scans')
self.assertEqual(ms_nspws, mms_nspws, 'Compare number of spws')
# Compare the scans
self.assertEqual(ms_scans.all(), mms_scans.all(),
'Compare all scan IDs')
try:
mdlocal1.open(msfile)
mdlocal2.open(self.mmsfile)
# Compare the spws
for i in ms_scans:
msi = mdlocal1.spwsforscan(i)
mmsi = mdlocal2.spwsforscan(i)
self.assertEqual(msi.all(), mmsi.all(),
'Compare spw Ids for a scan')
finally:
mdlocal1.close()
mdlocal2.close()
# Sort the output MSs so that they can be compared
myms = mstool()
myms.open(msfile)
myms.sort('ms_sorted.ms', [
'OBSERVATION_ID', 'ARRAY_ID', 'SCAN_NUMBER', 'FIELD_ID',
'DATA_DESC_ID', 'ANTENNA1', 'ANTENNA2', 'TIME'
])
myms.done()
myms.open(self.mmsfile)
myms.sort('mms_sorted.ms', [
'OBSERVATION_ID', 'ARRAY_ID', 'SCAN_NUMBER', 'FIELD_ID',
'DATA_DESC_ID', 'ANTENNA1', 'ANTENNA2', 'TIME'
])
myms.done()
# Ignore WEIGHT_SPECTRUM and SIGMA_SPECTRUM, which are empty columns
self.assertTrue(
th.compTables('ms_sorted.ms', 'mms_sorted.ms', [
'FLAG', 'FLAG_CATEGORY', 'TIME_CENTROID', 'WEIGHT_SPECTRUM',
'SIGMA_SPECTRUM', 'DATA'
]))
# Compare the DATA column
self.assertTrue(
th.compVarColTables('ms_sorted.ms', 'mms_sorted.ms', 'DATA'))
# The separation axis should be written to the output MMS
sepaxis = ph.axisType(self.mmsfile)
self.assertEqual(sepaxis, 'scan,spw',
'Partition did not write AxisType correctly in MMS')
def test_mms4(self):
'''test_mms4: Create MMS, lazy=True, with separationaxis=scan and scans selection in ASDM'''
retValue = {'success': True, 'msgs': "", 'error_msgs': ''}
myasdmname = 'uid___A002_X72bc38_X000'
themsname = myasdmname + ".ms"
# only the first 3 scans to save time
importasdm(myasdmname,
vis=themsname,
lazy=True,
scans='0:1~3',
createmms=True,
separationaxis='scan',
process_flags=False)
self.assertTrue(ParallelDataHelper.isParallelMS(themsname),
'Output is not a Multi-MS')
self.assertEqual(ph.axisType(themsname), 'scan',
'Separation axis of MMS should be scan')
print myname, ": Success! Now checking output ..."
mscomponents = set([
"ANTENNA/table.dat", "DATA_DESCRIPTION/table.dat",
"FEED/table.dat", "FIELD/table.dat", "FLAG_CMD/table.dat",
"HISTORY/table.dat", "OBSERVATION/table.dat", "POINTING/table.dat",
"POLARIZATION/table.dat", "PROCESSOR/table.dat",
"SOURCE/table.dat", "SPECTRAL_WINDOW/table.dat", "STATE/table.dat",
"SYSCAL/table.dat", "ANTENNA/table.f0",
"DATA_DESCRIPTION/table.f0", "FEED/table.f0", "FIELD/table.f0",
"FLAG_CMD/table.f0", "HISTORY/table.f0", "OBSERVATION/table.f0",
"POINTING/table.f0", "POLARIZATION/table.f0", "PROCESSOR/table.f0",
"SOURCE/table.f0", "SPECTRAL_WINDOW/table.f0", "STATE/table.f0",
"SYSCAL/table.f0"
])
for name in mscomponents:
if not os.access(themsname + "/" + name, os.F_OK):
print myname, ": Error ", themsname + "/" + name, "doesn't exist ..."
retValue['success'] = False
retValue['error_msgs'] = retValue[
'error_msgs'] + themsname + '/' + name + ' does not exist'
else:
print myname, ": ", name, "present."
print myname, ": MS exists. All tables present. Try opening as MS ..."
try:
mslocal.open(themsname)
except:
print myname, ": Error Cannot open MS table", themsname
retValue['success'] = False
retValue['error_msgs'] = retValue[
'error_msgs'] + 'Cannot open MS table ' + themsname
else:
mslocal.close()
print myname, ": OK. Checking tables in detail ..."
importasdm(asdm=myasdmname,
vis='reference.ms',
lazy=False,
overwrite=True,
scans='0:1~3',
createmms=True,
separationaxis='scan',
process_flags=False)
if (os.path.exists('reference.ms')):
retValue['success'] = th.checkwithtaql(
"select from [select from reference.ms orderby TIME, DATA_DESC_ID, ANTENNA1, ANTENNA2 ] t1, [select from "
+ themsname +
" orderby TIME, DATA_DESC_ID, ANTENNA1, ANTENNA2 ] t2 where (not all(near(t1.DATA,t2.DATA, 1.e-06)))"
) == 0
if not retValue['success']:
print "ERROR: DATA does not agree with reference."
else:
print "DATA columns agree."
retValueTmp = th.checkwithtaql(
"select from [select from reference.ms orderby TIME, DATA_DESC_ID, ANTENNA1, ANTENNA2 ] t1, [select from "
+ themsname +
" orderby TIME, DATA_DESC_ID, ANTENNA1, ANTENNA2 ] t2 where (not all(t1.FLAG==t2.FLAG)) "
) == 0
if not retValueTmp:
print "ERROR: FLAG does not agree with reference."
else:
print "FLAG columns agree."
retValue['success'] = retValue['success'] and retValueTmp
for subtname in [
"ANTENNA", "DATA_DESCRIPTION", "FEED", "FIELD",
"FLAG_CMD", "OBSERVATION", "POINTING", "POLARIZATION",
"PROCESSOR", "SOURCE", "SPECTRAL_WINDOW", "STATE",
"SYSCAL"
]:
print "\n*** Subtable ", subtname
excllist = []
if subtname == 'SOURCE':
excllist = [
'POSITION', 'TRANSITION', 'REST_FREQUENCY',
'SYSVEL'
]
if subtname == 'SYSCAL':
excllist = ['TANT_SPECTRUM', 'TANT_TSYS_SPECTRUM']
if subtname == 'SPECTRAL_WINDOW':
excllist = [
'CHAN_FREQ', 'CHAN_WIDTH', 'EFFECTIVE_BW',
'RESOLUTION'
]
for colname in excllist:
retValue['success'] = th.compVarColTables(
'reference.ms/SPECTRAL_WINDOW',
themsname + '/SPECTRAL_WINDOW', colname,
0.01) and retValue['success']
try:
retValue['success'] = th.compTables(
'reference.ms/' + subtname, themsname + '/' +
subtname, excllist, 0.01) and retValue['success']
except:
retValue['success'] = False
print "ERROR for table ", subtname
print retValue
self.assertTrue(retValue['success'], retValue['error_msgs'])
def listpartition(vis=None, createdict=None, listfile=None):
"""List information about an MMS data set in the logger:
Keyword arguments:
vis -- Name of multi-MS or normal MS.
default: none. example: vis='uidA002.mms'
createdict -- Create and return a dictionary with information about
the sub-MSs.
default: False
listfile -- save the output to a file
default: ''. Example: listfile="mylist.txt"
The task also returns a dictionary with scan summary information
for each sub-MS.
"""
casalog.origin('listpartition')
mslocal = casac.ms()
mslocal1 = casac.ms()
try:
if (type(vis) == str) & os.path.exists(vis):
mslocal.open(thems=vis)
else:
raise Exception, \
'Visibility data set not found - please verify the name'
# Check output filename existence
if listfile != '':
if (type(listfile) == str) & os.path.exists(listfile):
raise Exception, 'Output file \'%s\' already exists' % listfile
casalog.post('Will save output to \'%s\'' % listfile)
ffout = open(listfile, 'w')
# Is it a multi-MS?
ismms = mslocal.ismultims()
# List of MSs to process
mslist = []
# It is a multi-MS
if ismms:
mslist = mslocal.getreferencedtables()
mslist.sort()
sname = 'Sub-MS'
# Get the AxisType of the MMS
axis = ph.axisType(vis)
if axis == '':
axis = 'unknown'
casalog.post('This is a multi-MS with separation axis = ' + axis)
else:
mslist.append(vis)
sname = 'MS'
# Close top MS
mslocal.close()
# Get a consolidated dictionary with scan, spw, channel information
# of the list of subMSs. It adds the nrows of all sub-scans of a scan.
try:
outdict = {}
outdict = ph.getScanSpwSummary(mslist)
except Exception, instance:
casalog.post('%s' % instance, 'ERROR')
# Now loop through the dictionary to print the information
if outdict.keys() == []:
casalog.post('Error in processing dictionaries', 'ERROR')
indices = outdict.keys()
indices.sort()
counter = 0
for index in indices:
# Get data
MS = outdict[index]['MS']
SIZE = outdict[index]['size']
SCAN = outdict[index]['scanId']
# Sort scans for more optimal printing
# Print information per scan
firstscan = True
skeys = SCAN.keys()
skeys.sort()
for myscan in skeys:
SPW = outdict[index]['scanId'][myscan]['spwIds']
NCHAN = outdict[index]['scanId'][myscan]['nchans']
NROWS = outdict[index]['scanId'][myscan]['nrows']
# Get maximum widths
smxw = getWidth(outdict, 'spw')
cmxw = getWidth(outdict, 'channel')
# Create format
fhdr = '%-'+str(len(MS)+2)+'s' + '%-6s' + '%-'+str(smxw+2)+'s' + \
'%-'+str(cmxw+2)+'s' + '%-8s' + '%-6s'
# Print header
text = ''
if counter == 0:
text = text + fhdr % (sname, 'Scan', 'Spw', 'Nchan',
'Nrows', 'Size')
text = text + '\n'
counter += 1
# Print first scan
if firstscan:
text = text + fhdr % (MS, myscan, SPW, NCHAN, NROWS, SIZE)
else:
text = text + fhdr % ('', myscan, SPW, NCHAN, NROWS, '')
firstscan = False
# Print to a file
if listfile != '':
print >> ffout, text
else:
# Print to the logger
casalog.post(text)
if listfile != '':
ffout.close()
# Return the scan dictionary
if createdict:
return outdict
return {}
def listpartition(vis=None, createdict=None, listfile=None):
"""List information about an MMS data set in the logger:
Keyword arguments:
vis -- Name of multi-MS or normal MS.
default: none. example: vis='uidA002.mms'
createdict -- Create and return a dictionary with information about
the sub-MSs.
default: False
listfile -- save the output to a file
default: ''. Example: listfile="mylist.txt"
The task also returns a dictionary with scan summary information
for each sub-MS.
"""
casalog.origin('listpartition')
mslocal = casac.ms()
mslocal1 = casac.ms()
try:
if (type(vis) == str) & os.path.exists(vis):
mslocal.open(thems=vis)
else:
raise Exception, \
'Visibility data set not found - please verify the name'
# Check output filename existence
if listfile != '':
if (type(listfile) == str) & os.path.exists(listfile):
raise Exception, 'Output file \'%s\' already exists'%listfile
casalog.post('Will save output to \'%s\''%listfile)
ffout = open(listfile, 'w')
# Is it a multi-MS?
ismms = mslocal.ismultims()
# List of MSs to process
mslist = []
# It is a multi-MS
if ismms:
mslist = mslocal.getreferencedtables()
mslist.sort()
sname = 'Sub-MS'
# Get the AxisType of the MMS
axis = ph.axisType(vis)
if axis == '':
axis = 'unknown'
casalog.post('This is a multi-MS with separation axis = '+axis)
else:
mslist.append(vis)
sname = 'MS'
# Close top MS
mslocal.close()
# Get a consolidated dictionary with scan, spw, channel information
# of the list of subMSs. It adds the nrows of all sub-scans of a scan.
try:
outdict = {}
outdict = ph.getScanSpwSummary(mslist)
except Exception, instance:
casalog.post('%s'%instance,'ERROR')
# Now loop through the dictionary to print the information
if outdict.keys() == []:
casalog.post('Error in processing dictionaries','ERROR')
indices = outdict.keys()
indices.sort()
counter = 0
for index in indices:
# Get data
MS = outdict[index]['MS']
SIZE = outdict[index]['size']
SCAN = outdict[index]['scanId']
# Sort scans for more optimal printing
# Print information per scan
firstscan = True
skeys = SCAN.keys()
skeys.sort()
for myscan in skeys:
SPW = outdict[index]['scanId'][myscan]['spwIds']
NCHAN = outdict[index]['scanId'][myscan]['nchans']
NROWS = outdict[index]['scanId'][myscan]['nrows']
# Get maximum widths
smxw = getWidth(outdict, 'spw')
cmxw = getWidth(outdict, 'channel')
# Create format
fhdr = '%-'+str(len(MS)+2)+'s' + '%-6s' + '%-'+str(smxw+2)+'s' + \
'%-'+str(cmxw+2)+'s' + '%-8s' + '%-6s'
# Print header
text = ''
if counter == 0:
text = text + fhdr % (sname,'Scan','Spw','Nchan','Nrows','Size')
text = text + '\n'
counter += 1
# Print first scan
if firstscan:
text = text + fhdr % (MS, myscan, SPW, NCHAN, NROWS, SIZE)
else:
text = text + fhdr % ('', myscan, SPW, NCHAN, NROWS, '')
firstscan = False
# Print to a file
if listfile != '':
print >> ffout, text
else:
# Print to the logger
casalog.post(text)
if listfile != '':
ffout.close()
# Return the scan dictionary
if createdict:
return outdict
return {}
def test_default(self):
'''Partition: create an MMS with default values in parallel'''
# First split off one scan to run the test faster
split(vis=self.msfile, outputvis='split30.ms', datacolumn='DATA', scan='30')
msfile = 'split30.ms'
partition(vis=msfile, outputvis=self.mmsfile)
self.assertTrue(os.path.exists(self.mmsfile), 'MMS was not created for this test')
# Gather several metadata information
# for the MS
mdlocal1 = msmdtool()
mdlocal1.open(msfile)
ms_rows = mdlocal1.nrows()
ms_nscans = mdlocal1.nscans()
ms_nspws = mdlocal1.nspw()
ms_scans = mdlocal1.scannumbers()
mdlocal1.close()
# for the MMS
mdlocal2 = msmdtool()
mdlocal2.open(self.mmsfile)
mms_rows = mdlocal2.nrows()
mms_nscans = mdlocal2.nscans()
mms_nspws = mdlocal2.nspw()
mms_scans = mdlocal2.scannumbers()
mdlocal2.close()
# Compare the number of rows
self.assertEqual(ms_rows, mms_rows, 'Compare total number of rows in MS and MMS')
self.assertEqual(ms_nscans, mms_nscans, 'Compare number of scans')
self.assertEqual(ms_nspws, mms_nspws, 'Compare number of spws')
# Compare the scans
self.assertEqual(ms_scans.all(), mms_scans.all(), 'Compare all scan IDs')
try:
mdlocal1.open(msfile)
mdlocal2.open(self.mmsfile)
# Compare the spws
for i in ms_scans:
msi = mdlocal1.spwsforscan(i)
mmsi = mdlocal2.spwsforscan(i)
self.assertEqual(msi.all(), mmsi.all(), 'Compare spw Ids for a scan')
finally:
mdlocal1.close()
mdlocal2.close()
# Sort the output MSs so that they can be compared
myms = mstool()
myms.open(msfile)
myms.sort('ms_sorted.ms',['OBSERVATION_ID','ARRAY_ID','SCAN_NUMBER','FIELD_ID','DATA_DESC_ID','ANTENNA1','ANTENNA2','TIME'])
myms.done()
myms.open(self.mmsfile)
myms.sort('mms_sorted.ms',['OBSERVATION_ID','ARRAY_ID','SCAN_NUMBER','FIELD_ID','DATA_DESC_ID','ANTENNA1','ANTENNA2','TIME'])
myms.done()
# Ignore WEIGHT_SPECTRUM and SIGMA_SPECTRUM, which are empty columns
self.assertTrue(th.compTables('ms_sorted.ms', 'mms_sorted.ms',
['FLAG','FLAG_CATEGORY','TIME_CENTROID',
'WEIGHT_SPECTRUM','SIGMA_SPECTRUM','DATA']))
# Compare the DATA column
self.assertTrue(th.compVarColTables('ms_sorted.ms', 'mms_sorted.ms','DATA'))
# The separation axis should be written to the output MMS
sepaxis = ph.axisType(self.mmsfile)
self.assertEqual(sepaxis, 'scan,spw', 'Partition did not write AxisType correctly in MMS')
def test_default(self):
"""Partition: create an MMS with default values in parallel"""
# First split off one scan to run the test faster
split(vis=self.msfile, outputvis="split30.ms", datacolumn="DATA", scan="30")
msfile = "split30.ms"
partition(vis=msfile, outputvis=self.mmsfile)
self.assertTrue(os.path.exists(self.mmsfile), "MMS was not created for this test")
# Gather several metadata information
# for the MS
mdlocal1 = msmdtool()
mdlocal1.open(msfile)
ms_rows = mdlocal1.nrows()
ms_nscans = mdlocal1.nscans()
ms_nspws = mdlocal1.nspw()
ms_scans = mdlocal1.scannumbers()
mdlocal1.close()
# for the MMS
mdlocal2 = msmdtool()
mdlocal2.open(self.mmsfile)
mms_rows = mdlocal2.nrows()
mms_nscans = mdlocal2.nscans()
mms_nspws = mdlocal2.nspw()
mms_scans = mdlocal2.scannumbers()
mdlocal2.close()
# Compare the number of rows
self.assertEqual(ms_rows, mms_rows, "Compare total number of rows in MS and MMS")
self.assertEqual(ms_nscans, mms_nscans, "Compare number of scans")
self.assertEqual(ms_nspws, mms_nspws, "Compare number of spws")
# Compare the scans
self.assertEqual(ms_scans.all(), mms_scans.all(), "Compare all scan IDs")
try:
mdlocal1.open(msfile)
mdlocal2.open(self.mmsfile)
# Compare the spws
for i in ms_scans:
msi = mdlocal1.spwsforscan(i)
mmsi = mdlocal2.spwsforscan(i)
self.assertEqual(msi.all(), mmsi.all(), "Compare spw Ids for a scan")
finally:
mdlocal1.close()
mdlocal2.close()
# Sort the output MSs so that they can be compared
myms = mstool()
myms.open(msfile)
myms.sort(
"ms_sorted.ms",
["OBSERVATION_ID", "ARRAY_ID", "SCAN_NUMBER", "FIELD_ID", "DATA_DESC_ID", "ANTENNA1", "ANTENNA2", "TIME"],
)
myms.done()
myms.open(self.mmsfile)
myms.sort(
"mms_sorted.ms",
["OBSERVATION_ID", "ARRAY_ID", "SCAN_NUMBER", "FIELD_ID", "DATA_DESC_ID", "ANTENNA1", "ANTENNA2", "TIME"],
)
myms.done()
self.assertTrue(
th.compTables(
"ms_sorted.ms", "mms_sorted.ms", ["FLAG", "FLAG_CATEGORY", "TIME_CENTROID", "WEIGHT_SPECTRUM", "DATA"]
)
)
# Compare the DATA column
self.assertTrue(th.compVarColTables("ms_sorted.ms", "mms_sorted.ms", "DATA"))
# The separation axis should be written to the output MMS
sepaxis = ph.axisType(self.mmsfile)
self.assertEqual(sepaxis, "scan,spw", "Partition did not write AxisType correctly in MMS")
def test_mms4(self):
'''test_mms4: Create MMS, lazy=True, with separationaxis=scan and scans selection in ASDM'''
retValue = {'success': True, 'msgs': "", 'error_msgs': '' }
myasdmname = 'uid___A002_X72bc38_X000'
themsname = myasdmname + ".ms"
# only the first 3 scans to save time
importasdm(myasdmname, vis=themsname, lazy=True, scans='0:1~3', createmms=True, separationaxis='scan',
process_flags=False)
self.assertTrue(ParallelDataHelper.isParallelMS(themsname), 'Output is not a Multi-MS')
self.assertEqual(ph.axisType(themsname), 'scan', 'Separation axis of MMS should be scan')
print myname, ": Success! Now checking output ..."
mscomponents = set(["ANTENNA/table.dat",
"DATA_DESCRIPTION/table.dat",
"FEED/table.dat",
"FIELD/table.dat",
"FLAG_CMD/table.dat",
"HISTORY/table.dat",
"OBSERVATION/table.dat",
"POINTING/table.dat",
"POLARIZATION/table.dat",
"PROCESSOR/table.dat",
"SOURCE/table.dat",
"SPECTRAL_WINDOW/table.dat",
"STATE/table.dat",
"SYSCAL/table.dat",
"ANTENNA/table.f0",
"DATA_DESCRIPTION/table.f0",
"FEED/table.f0",
"FIELD/table.f0",
"FLAG_CMD/table.f0",
"HISTORY/table.f0",
"OBSERVATION/table.f0",
"POINTING/table.f0",
"POLARIZATION/table.f0",
"PROCESSOR/table.f0",
"SOURCE/table.f0",
"SPECTRAL_WINDOW/table.f0",
"STATE/table.f0",
"SYSCAL/table.f0"
])
for name in mscomponents:
if not os.access(themsname+"/"+name, os.F_OK):
print myname, ": Error ", themsname+"/"+name, "doesn't exist ..."
retValue['success']=False
retValue['error_msgs']=retValue['error_msgs']+themsname+'/'+name+' does not exist'
else:
print myname, ": ", name, "present."
print myname, ": MS exists. All tables present. Try opening as MS ..."
try:
mslocal.open(themsname)
except:
print myname, ": Error Cannot open MS table", themsname
retValue['success']=False
retValue['error_msgs']=retValue['error_msgs']+'Cannot open MS table '+themsname
else:
mslocal.close()
print myname, ": OK. Checking tables in detail ..."
importasdm(asdm=myasdmname, vis='reference.ms', lazy=False, overwrite=True, scans='0:1~3',
createmms=True, separationaxis='scan', process_flags=False)
if(os.path.exists('reference.ms')):
retValue['success'] = th.checkwithtaql("select from [select from reference.ms orderby TIME, DATA_DESC_ID, ANTENNA1, ANTENNA2 ] t1, [select from "
+themsname+" orderby TIME, DATA_DESC_ID, ANTENNA1, ANTENNA2 ] t2 where (not all(near(t1.DATA,t2.DATA, 1.e-06)))") == 0
if not retValue['success']:
print "ERROR: DATA does not agree with reference."
else:
print "DATA columns agree."
retValueTmp = th.checkwithtaql("select from [select from reference.ms orderby TIME, DATA_DESC_ID, ANTENNA1, ANTENNA2 ] t1, [select from "
+themsname+" orderby TIME, DATA_DESC_ID, ANTENNA1, ANTENNA2 ] t2 where (not all(t1.FLAG==t2.FLAG)) ") == 0
if not retValueTmp:
print "ERROR: FLAG does not agree with reference."
else:
print "FLAG columns agree."
retValue['success'] = retValue['success'] and retValueTmp
for subtname in ["ANTENNA",
"DATA_DESCRIPTION",
"FEED",
"FIELD",
"FLAG_CMD",
"OBSERVATION",
"POINTING",
"POLARIZATION",
"PROCESSOR",
"SOURCE",
"SPECTRAL_WINDOW",
"STATE",
"SYSCAL"]:
print "\n*** Subtable ",subtname
excllist = []
if subtname=='SOURCE':
excllist=['POSITION', 'TRANSITION', 'REST_FREQUENCY', 'SYSVEL']
if subtname=='SYSCAL':
excllist=['TANT_SPECTRUM', 'TANT_TSYS_SPECTRUM']
if subtname=='SPECTRAL_WINDOW':
excllist=['CHAN_FREQ', 'CHAN_WIDTH', 'EFFECTIVE_BW', 'RESOLUTION']
for colname in excllist:
retValue['success'] = th.compVarColTables('reference.ms/SPECTRAL_WINDOW',
themsname+'/SPECTRAL_WINDOW', colname, 0.01) and retValue['success']
try:
retValue['success'] = th.compTables('reference.ms/'+subtname,
themsname+'/'+subtname, excllist,
0.01) and retValue['success']
except:
retValue['success'] = False
print "ERROR for table ", subtname
print retValue
self.assertTrue(retValue['success'],retValue['error_msgs'])
def pieflag(vis,
field, # data selection parameters
refchanfile,
fitorder_RR_LL,
fitorder_RL_LR,
scalethresh,
SEFDfile, # scalethresh parameter
plotSEFD,
dynamicflag,
chunktime, # dynamicflag parameters
stdmax,
maxoffset,
staticflag,
madmax, # staticflag parameter
binsamples,
extendflag,
boxtime, # extendflag parameters
boxthresh):
#
# Task pieflag
# Flags bad data by comparing with clean channels in bandpass-calibrated data.
#
# Original reference: E. Middelberg, 2006, PASA, 23, 64
# Rewritten for use in CASA and updated to account for wideband
# and SEFD effects by Christopher A. Hales 2014.
#
# Thanks to Kumar Golap, Justo Gonzalez, Jeff Kern, James Robnett,
# Urvashi Rau, Sanjay Bhatnagar, and of course Enno Middelberg
# for expert advice. Thanks to Emmanuel Momjian for providing
# Jansky VLA SEFD data for L and X bands (EVLA Memos 152 and 166)
# and to Bryan Butler for providing access to all other bands
# from the Jansky VLA Exposure Calculator.
#
# Version 4.4 released 26 October 2016
# Tested with CASA 4.7.0 using Jansky VLA data
# Available at: http://github.com/chrishales/pieflag
#
# Reference for this version:
# C. A. Hales, E. Middelberg, 2014, Astrophysics Source Code Library, 1408.014
# http://adsabs.harvard.edu/abs/2014ascl.soft08014H
#
startTime = time.time()
casalog.origin('pieflag')
casalog.post('--> pieflag version 4.4')
if (not staticflag) and (not dynamicflag):
casalog.post('*** ERROR: You need to select static or dynamic flagging.', 'ERROR')
casalog.post('*** ERROR: Exiting pieflag.', 'ERROR')
return
ms.open(vis)
vis=ms.name()
ms.close()
useMPI = MPIEnvironment.is_mpi_enabled
if useMPI:
if vis.lower().endswith('.ms'):
useMPI=False
casalog.post('--> MS will be processed in serial mode.')
elif ph.axisType(vis) == 'baseline':
# client is ID 0 and will not perform parallel processing, servers start from ID 1
nthreads = MPIEnvironment.rank
subms_path = vis+'/SUBMSS/'
subms = filter(lambda x: os.path.isdir(os.path.join(subms_path, x)), os.listdir(subms_path))
if len(subms) != nthreads:
casalog.post('*** ERROR: Mismatch, MMS tailored for '+str(len(subms))+' engines but '+\
'CASA session tailored for '+str(nthreads)+' engines.', 'ERROR')
casalog.post('*** ERROR: Exiting pieflag.', 'ERROR')
return
server_list = MPIEnvironment.mpi_server_rank_list()
casalog.post('--> Initializing MPI parallel cluster with '+str(nthreads)+' engines.')
client = MPICommandClient()
client.start_services()
# do some detective work to find appropriate path to push to clients
syspaths = sys.path
n = 0
for k in range(len(syspaths)):
if os.path.isfile(syspaths[k]+'/mytasks.py'):
for line in open(syspaths[k]+'/mytasks.py','r'):
if re.search("task_location\['pieflag'\]",line):
if n==0:
n += 1
addpath = syspaths[k]
elif syspaths[k] != addpath:
n += 1
if n == 1:
casalog.filter('WARN')
#client.set_log_level('WARN')
client.push_command_request("casalog.filter('WARN')",True,server_list)
client.push_command_request("sys.path.append('"+addpath+"')",True,server_list)
client.push_command_request('from task_pieflag import pieflag_getflagstats',True,server_list)
client.push_command_request('from task_pieflag import pieflag_flag',True,server_list)
casalog.filter('INFO')
else:
if n == 0:
casalog.post('*** ERROR: pieflag mytasks.py installation not found in sys.path', 'ERROR')
else:
casalog.post('*** ERROR: Ambiguity, sys.path contains more than 1 pieflag installation', 'ERROR')
casalog.post('*** (pieflag referenced in '+str(n)+' unique path/mytasks.py)', 'ERROR')
casalog.post('*** ERROR: Exiting pieflag.', 'ERROR')
return
fcall1 = 'pieflag_getflagstats(vis,field,spw,npol,feedbasis)'
fcall2 = 'pieflag_flag(vis,datacol,nthreads,field,vtbleLIST,inttime,nant,ddid,spw,refchan,nchan,npol,'+\
'feedbasis,fitorderLIST,sefdLIST,staticflag,madmax,binsamples,dynamicflag,chunktime,stdmax,'+\
'maxoffset,extendflag,boxtime,boxthresh)'
else:
casalog.post('*** ERROR: MMS is not partitioned by baseline. Cannot process.', 'ERROR')
casalog.post('*** Use partition() to revert to MS then create baseline MMS.', 'ERROR')
casalog.post('*** ERROR: Exiting pieflag.', 'ERROR')
return
else:
if vis.lower().endswith('.mms'):
casalog.post('*** ERROR: pieflag cannot handle MMS in non-MPI-enabled CASA session.', 'ERROR')
casalog.post('*** ERROR: Exiting pieflag.', 'ERROR')
return
else:
casalog.post('--> MS will be processed in serial mode.')
tb.open(vis)
if any('CORRECTED_DATA' in colnames for colnames in tb.colnames()):
datacol='CORRECTED_DATA'
else:
datacol='DATA'
tb.close()
# load in reference channel details
# OK, there are probably more elegant ways
# of implementing the following code...meh
refchandict=json.load(open(refchanfile))
spw=[]
for i in refchandict.keys():
spw.append(int(i))
nspw=len(spw)
# json doesn't seem to load in the spw order properly
# The user might not have entered spw's in order either
# so perform sort just in case
# note: no need to perform sort on the string versions
spw.sort()
# now get reference channels in corresponding sorted order
refchan=[]
for i in range(nspw):
refchan.append(refchandict[str(spw[i])])
# open MS and select relevant data
ms.open(vis)
ms.msselect({'field':str(field)})
# get integration time
scan_summary = ms.getscansummary()
ms.close()
scan_list = []
for scan in scan_summary:
if scan_summary[scan]['0']['FieldId'] == field:
scan_list.append(int(scan))
inttime=scan_summary[str(scan_list[0])]['0']['IntegrationTime']
# get around potential floating point issues by rounding to nearest 1e-5 seconds
if inttime != round(inttime,5):
casalog.post('*** WARNING: It seems your integration time is specified to finer than 1e-5 seconds.','WARN')
casalog.post('*** pieflag will assume this is a rounding error and carry on.','WARN')
for i in range(len(scan_list)):
if round(inttime,5) != round(scan_summary[str(scan_list[i])]['0']['IntegrationTime'],5):
casalog.post('*** ERROR: Bummer, pieflag is not set up to handle '+\
'changing integration times throughout your MS.', 'ERROR')
casalog.post('*** ERROR: Exiting pieflag.','ERROR')
return
# get number of baselines
tb.open(vis+'/ANTENNA')
atble=tb.getcol('NAME')
tb.close()
nant=atble.shape[0]
nbaselines=nant*(nant-1)/2
# channel to frequency (Hz) conversion
tb.open(vis+'/SPECTRAL_WINDOW')
vtble=tb.getcol('CHAN_FREQ')
tb.close()
# vtble format is vtble[channel][spw]
# assume each spw has the same number of channels
nchan=vtble.shape[0]
# check that spw frequencies increase monotonically
spwcheck=vtble[0,0]
for s in range(1,len(vtble[0,:])):
if vtble[0,s]<spwcheck:
casalog.post("*** ERROR: Your spw's are not ordered with increasing frequency.",'ERROR')
casalog.post('*** ERROR: Consider splitting your data and restarting pieflag. Exiting','ERROR')
return
spwcheck=vtble[0,s]
# get number of polarizations, assume they don't change throughout observation
# get details from the first user-selected spw within the first scan on target field
# note: I won't assume that spw specifies data_desc_id in the main table, even
# though in most cases it probably does. Probably overkill given the lack
# of checks done elsewhere in this code...
tb.open(vis+'/DATA_DESCRIPTION')
temptb=tb.query('SPECTRAL_WINDOW_ID='+str(spw[0]))
# while here, get the data_desc_id values that pair with spw number
tempddid=tb.getcol('SPECTRAL_WINDOW_ID').tolist()
ddid=[]
for s in range(nspw):
ddid.append(tempddid.index(spw[s]))
tb.close()
polid=temptb.getcell('POLARIZATION_ID')
tb.open(vis+'/POLARIZATION')
npol=tb.getcell('NUM_CORR',polid)
poltype=tb.getcell('CORR_TYPE',polid)
tb.close()
if not (npol == 2 or npol == 4):
casalog.post('*** ERROR: Your data contains '+str(npol)+' polarization products.','ERROR')
casalog.post('*** ERROR: pieflag can only handle 2 (eg RR/LL) or 4 (eg RR/RL/LR/LL). Exiting.','ERROR')
return
# see stokes.h for details
if poltype[0] == 5:
# circular
feedbasis = 1
elif poltype[0] == 9:
#linear
feedbasis = 0
else:
casalog.post('*** ERROR: Your data uses an unsupported feed basis. Exiting','ERROR')
return
casalog.post('--> Some details about your data:')
casalog.post(' data column to process = '+datacol)
casalog.post(' integration time = '+str(inttime)+' sec')
casalog.post(' number of baselines = '+str(nbaselines))
casalog.post(' spectral windows to process = '+str(spw))
casalog.post(' number of channels per spectral window = '+str(nchan))
if feedbasis:
casalog.post(' feed basis = circular')
else:
casalog.post(' feed basis = linear')
casalog.post(' number of polarization products to process = '+str(npol))
casalog.post('--> Statistics of pre-existing flags:')
flag0 = np.zeros((nspw,2*npol+2))
for i in range(nspw):
casalog.filter('WARN')
if useMPI:
for k in range(nthreads):
param = {'vis':vis+'/SUBMSS/'+subms[k],'field':field,\
'spw':spw[i],'npol':npol,'feedbasis':feedbasis}
if k == 0:
pid = client.push_command_request(fcall1,False,None,param)
else:
pid.append((client.push_command_request(fcall1,False,None,param))[0])
presults = client.get_command_response(pid,True)
for k in range(nthreads):
flag0[i] += presults[k]['ret']
else:
flag0[i] = pieflag_getflagstats(vis,field,spw[i],npol,feedbasis)
casalog.filter('INFO')
RRs="{:.1f}".format(flag0[i][0]/flag0[i][1]*100.)
LLs="{:.1f}".format(flag0[i][2]/flag0[i][3]*100.)
TOTs="{:.1f}".format(flag0[i][4]/flag0[i][5]*100.)
if npol == 2:
if feedbasis:
outstr=' flagged data in spw='+str(spw[i])+': RR='+RRs+'% LL='+LLs+'% total='+TOTs+'%'
else:
outstr=' flagged data in spw='+str(spw[i])+': XX='+RRs+'% YY='+LLs+'% total='+TOTs+'%'
else:
RLs="{:.1f}".format(flag0[i][6]/flag0[i][7]*100.)
LRs="{:.1f}".format(flag0[i][8]/flag0[i][9]*100.)
if feedbasis:
outstr=' flagged data in spw='+str(spw[i])+': RR='+RRs+'% RL='+RLs+'% LR='+LRs+'% LL='+LLs+'% total='+TOTs+'%'
else:
outstr=' flagged data in spw='+str(spw[i])+': XX='+RRs+'% XY='+RLs+'% YX='+LRs+'% YY='+LLs+'% total='+TOTs+'%'
casalog.post(outstr)
# Check there are enough spectral windows to perform the fitting later on. If not, lower the order.
if fitorder_RR_LL > nspw-1:
if fitorder_RR_LL == 2:
if feedbasis:
casalog.post('*** WARNING: pieflag needs at least 3 spectral windows to fit for RR or LL spectral curvature.','WARN')
else:
casalog.post('*** WARNING: pieflag needs at least 3 spectral windows to fit for XX or YY spectral curvature.','WARN')
else:
if feedbasis:
casalog.post('*** WARNING: pieflag needs at least 2 spectral windows to fit for RR or LL spectral index.','WARN')
else:
casalog.post('*** WARNING: pieflag needs at least 2 spectral windows to fit for XX or YY spectral index.','WARN')
if nspw == 2:
fitorder_RR_LL=1
else:
fitorder_RR_LL=0
casalog.post('*** WARNING: fitorder_RR_LL has been reduced to '+str(int(fitorder_RR_LL))+ ' and','WARN')
casalog.post('*** may be reduced further for some baselines if the','WARN')
casalog.post('*** reference channel isn\'t available in all selected spw\'s.','WARN')
if npol == 2:
fitorder = np.zeros(2)
fitorder[0] = fitorder_RR_LL
fitorder[1] = fitorder_RR_LL
elif npol == 4:
if fitorder_RL_LR > nspw-1:
if fitorder_RL_LR == 2:
casalog.post('*** WARNING: pieflag needs at least 3 spectral windows to fit for RL or LR spectral curvature.','WARN')
else:
casalog.post('*** WARNING: pieflag needs at least 2 spectral windows to fit for RL or LR spectral index.','WARN')
if nspw == 2:
fitorder_RL_LR=1
else:
fitorder_RL_LR=0
casalog.post('*** WARNING: fitorder_RL_LR has been reduced to '+str(int(fitorder_RL_LR))+' and','WARN')
casalog.post('*** may be reduced further for some baselines if the','WARN')
casalog.post('*** reference channel isn\'t available in all selected spw\'s.','WARN')
fitorder = np.zeros(4)
fitorder[0] = fitorder_RR_LL
fitorder[1] = fitorder_RL_LR
fitorder[2] = fitorder_RL_LR
fitorder[3] = fitorder_RR_LL
if scalethresh:
# read in SEFD data and interpolate to get values at our channel frequencies
casalog.post('--> Reading in SEFD and interpolating at channel frequencies...')
sefdRAW=np.loadtxt(SEFDfile)
sefd=np.zeros((nspw,nchan))
if not np.all(np.diff(sefdRAW[:,0]) >= 0):
casalog.post('*** ERROR: Your SEFD file must be in order of increasing frequency.','ERROR')
casalog.post('*** ERROR: Exiting pieflag.','ERROR')
return
for i in range(nspw):
if (vtble[:,spw[i]].min() < sefdRAW[:,0].min()) or (vtble[:,spw[i]].max() > sefdRAW[:,0].max()):
casalog.post('*** ERROR: pieflag cannot extrapolate your SEFD.','ERROR')
casalog.post('*** ERROR: Provide new SEFD covering your entire frequency range.','ERROR')
casalog.post('*** ERROR: Exiting pieflag.','ERROR')
return
sefdINTERP = interp1d(sefdRAW[:,0],sefdRAW[:,1])
for i in range(nspw):
sefdREFCHAN = sefdINTERP(vtble[refchan[i]][spw[i]])
for j in range(nchan):
# values in each spectral window will be relative to the reference channel value
sefd[i][j] = sefdINTERP(vtble[j][spw[i]]) / sefdREFCHAN
if plotSEFD:
# clunky, but works, meh...
sefdPLOT=np.zeros((nspw*nchan,3))
k=0
for i in range(nspw):
sefdREFCHAN = sefdINTERP(vtble[refchan[i]][spw[i]])
for j in range(nchan):
sefdPLOT[k][0] = vtble[j][spw[i]]/1.0e9
sefdPLOT[k][1] = sefd[i][j] * sefdREFCHAN
sefdPLOT[k][2] = sefd[i][j]
k += 1
f, (ax1, ax2) = plt.subplots(2,sharex=True)
ax1.plot(sefdRAW[:,0]/1.0e9,sefdRAW[:,1],'b-',sefdPLOT[:,0],sefdPLOT[:,1],'r.',markersize=10)
ax2.plot([sefdRAW[0,0]/1.0e9,sefdRAW[len(sefdRAW[:,0])-1,0]/1.0e9],[1.,1.],'c-',sefdPLOT[:,0],sefdPLOT[:,2],'r.',markersize=10)
f.subplots_adjust(hspace=0)
plt.setp([a.get_xticklabels() for a in f.axes[:-1]], visible=False)
ax1.set_title('relative sensitivity assumed across your band,\nnormalized to the reference channel in each spw')
ax1.legend(['raw input','interpolated'])
ax1.set_ylabel('SEFD (arbitrary units)')
ax2.set_xlabel('frequency (GHz)')
ax2.set_ylabel('SEFD (normalized units per spw)')
else:
sefd=np.ones((nspw,nchan))
if not staticflag:
madmax = 0
binsamples = 0
if not dynamicflag:
chunktime = 0
stdmax = 0
maxoffset = 0
if not extendflag:
boxtime = 0
boxthresh = 0
# forcibly remove all lock files
#os.system('find '+vis+' -name "*lock" -print | xargs rm')
if useMPI:
casalog.post('--> pieflag will now flag your data using '+str(nthreads)+' parallel threads.')
casalog.filter('WARN')
for k in range(nthreads):
param = {'vis':vis+'/SUBMSS/'+subms[k],'datacol':datacol,'nthreads':nthreads,'field':field,
'vtbleLIST':vtble.tolist(),'inttime':inttime,'nant':nant,
'ddid':ddid,'spw':spw,'refchan':refchan,'nchan':nchan,'npol':npol,'feedbasis':feedbasis,
'fitorderLIST':fitorder.tolist(),'sefdLIST':sefd.tolist(),
'staticflag':staticflag,'madmax':madmax,'binsamples':binsamples,
'dynamicflag':dynamicflag,'chunktime':chunktime,'stdmax':stdmax,'maxoffset':maxoffset,
'extendflag':extendflag,'boxtime':boxtime,'boxthresh':boxthresh}
if k == 0:
pid = client.push_command_request(fcall2,False,None,param)
else:
pid.append((client.push_command_request(fcall2,False,None,param))[0])
presults = client.get_command_response(pid,True)
casalog.filter('INFO')
else:
casalog.post('--> pieflag will now flag your data in serial mode.')
pieflag_flag(vis,datacol,1,field,
vtble.tolist(),inttime,nant,
ddid,spw,refchan,nchan,npol,feedbasis,
fitorder.tolist(),sefd.tolist(),
staticflag,madmax,binsamples,
dynamicflag,chunktime,stdmax,maxoffset,
extendflag,boxtime,boxthresh)
# show updated flagging statistics
casalog.post('--> Statistics of final flags (including pre-existing):')
flag1 = np.zeros((nspw,2*npol+2))
for i in range(nspw):
casalog.filter('WARN')
if useMPI:
for k in range(nthreads):
param = {'vis':vis+'/SUBMSS/'+subms[k],'field':field,\
'spw':spw[i],'npol':npol,'feedbasis':feedbasis}
if k == 0:
pid = client.push_command_request(fcall1,False,None,param)
else:
pid.append((client.push_command_request(fcall1,False,None,param))[0])
presults = client.get_command_response(pid,True)
for k in range(nthreads):
flag1[i] += presults[k]['ret']
else:
flag1[i] = pieflag_getflagstats(vis,field,spw[i],npol,feedbasis)
casalog.filter('INFO')
RRs="{:.1f}".format(flag1[i][0]/flag1[i][1]*100.)
LLs="{:.1f}".format(flag1[i][2]/flag1[i][3]*100.)
TOTs="{:.1f}".format(flag1[i][4]/flag1[i][5]*100.)
if npol == 2:
if feedbasis:
outstr=' flagged data in spw='+str(spw[i])+': RR='+RRs+'% LL='+LLs+'% total='+TOTs+'%'
else:
outstr=' flagged data in spw='+str(spw[i])+': XX='+RRs+'% YY='+LLs+'% total='+TOTs+'%'
else:
RLs="{:.1f}".format(flag1[i][6]/flag1[i][7]*100.)
LRs="{:.1f}".format(flag1[i][8]/flag1[i][9]*100.)
if feedbasis:
outstr=' flagged data in spw='+str(spw[i])+': RR='+RRs+'% RL='+RLs+'% LR='+LRs+'% LL='+LLs+'% total='+TOTs+'%'
else:
outstr=' flagged data in spw='+str(spw[i])+': XX='+RRs+'% XY='+RLs+'% YX='+LRs+'% YY='+LLs+'% total='+TOTs+'%'
casalog.post(outstr)
casalog.post('--> Statistics of pieflag flags (excluding pre-existing):')
for i in range(nspw):
RRs="{:.1f}".format((flag1[i][0]-flag0[i][0])/flag0[i][1]*100.)
LLs="{:.1f}".format((flag1[i][2]-flag0[i][2])/flag0[i][3]*100.)
TOTs="{:.1f}".format((flag1[i][4]-flag0[i][4])/flag0[i][5]*100.)
if npol == 2:
if feedbasis:
outstr=' data flagged in spw='+str(spw[i])+': RR='+RRs+'% LL='+LLs+'% total='+TOTs+'%'
else:
outstr=' data flagged in spw='+str(spw[i])+': XX='+RRs+'% YY='+LLs+'% total='+TOTs+'%'
else:
RLs="{:.1f}".format((flag1[i][6]-flag0[i][6])/flag0[i][7]*100.)
LRs="{:.1f}".format((flag1[i][8]-flag0[i][8])/flag0[i][9]*100.)
if feedbasis:
outstr=' data flagged in spw='+str(spw[i])+': RR='+RRs+'% RL='+RLs+'% LR='+LRs+'% LL='+LLs+'% total='+TOTs+'%'
else:
outstr=' data flagged in spw='+str(spw[i])+': XX='+RRs+'% XY='+RLs+'% YX='+LRs+'% YY='+LLs+'% total='+TOTs+'%'
casalog.post(outstr)
# forcibly remove all lock files
#os.system('find '+vis+' -name "*lock" -print | xargs rm')
if useMPI:
#client.set_log_level('INFO')
client.push_command_request("casalog.filter('INFO')",True,server_list)
t=time.time()-startTime
casalog.post('--> pieflag run time: '+str(int(t//3600))+' hours '+\
str(int(t%3600//60))+' minutes '+str(int(t%60))+' seconds')
