def test_nomms(self):
"""Partition: Create a normal MS with createmms=False"""
partition(vis=self.msfile, outputvis=self.mmsfile, createmms=False)
self.assertTrue(os.path.exists(self.mmsfile), "MMS was not created for this test")
# Sort the output MSs so that they can be compared
myms = mstool()
myms.open(self.msfile)
myms.sort(
"ms_sorted.ms",
["OBSERVATION_ID", "ARRAY_ID", "SCAN_NUMBER", "FIELD_ID", "DATA_DESC_ID", "ANTENNA1", "ANTENNA2", "TIME"],
)
myms.done()
myms.open(self.mmsfile)
myms.sort(
"mms_sorted.ms",
["OBSERVATION_ID", "ARRAY_ID", "SCAN_NUMBER", "FIELD_ID", "DATA_DESC_ID", "ANTENNA1", "ANTENNA2", "TIME"],
)
myms.done()
# Compare both tables. Ignore the DATA column and compare it in next line
self.assertTrue(
th.compTables("ms_sorted.ms", "mms_sorted.ms", ["FLAG_CATEGORY", "FLAG", "WEIGHT_SPECTRUM", "DATA"])
)
# Compare the DATA column
self.assertTrue(th.compVarColTables("ms_sorted.ms", "mms_sorted.ms", "DATA"))
def test_MMS1(self):
'''mstransform: input MMS should be the same as output MMS'''
# Create an MMS in the setup
self.createMMS(self.vis, axis='scan', spws='0,1')
# Create another MS and compare. They should be the same
self.outputms = 'thesame.mms'
mstransform(vis=self.testmms, outputvis=self.outputms, datacolumn='data')
self.assertTrue(ParallelDataHelper.isParallelMS(self.outputms),'Output is not an MMS')
# Sort the MSs so that they can be compared
myms = mstool()
myms.open(self.testmms)
myms.sort('input_sorted.ms',['OBSERVATION_ID','ARRAY_ID','SCAN_NUMBER','FIELD_ID','DATA_DESC_ID','ANTENNA1','ANTENNA2','TIME'])
myms.done()
myms.open(self.outputms)
myms.sort('output_sorted.ms',['OBSERVATION_ID','ARRAY_ID','SCAN_NUMBER','FIELD_ID','DATA_DESC_ID','ANTENNA1','ANTENNA2','TIME'])
myms.done()
# Compare both tables. Ignore the DATA column and compare it in next line
self.assertTrue(th.compTables('input_sorted.ms','output_sorted.ms',
['FLAG_CATEGORY','FLAG','WEIGHT_SPECTRUM','SIGMA_SPECTRUM','DATA']))
# Compare the DATA column
self.assertTrue(th.compVarColTables('input_sorted.ms','output_sorted.ms','DATA'))
# The separation axis should be copied to the output MMS
in_sepaxis = ph.axisType(self.testmms)
out_sepaxis = ph.axisType(self.outputms)
self.assertEqual(in_sepaxis, out_sepaxis, 'AxisTypes from input and output MMS do not match')
def test_nomms(self):
'''Partition: Create a normal MS with createmms=False'''
partition(vis=self.msfile, outputvis=self.mmsfile, createmms=False)
self.assertTrue(os.path.exists(self.mmsfile),
'MMS was not created for this test')
# Sort the output MSs so that they can be compared
myms = mstool()
myms.open(self.msfile)
myms.sort('ms_sorted.ms', [
'OBSERVATION_ID', 'ARRAY_ID', 'SCAN_NUMBER', 'FIELD_ID',
'DATA_DESC_ID', 'ANTENNA1', 'ANTENNA2', 'TIME'
])
myms.done()
myms.open(self.mmsfile)
myms.sort('mms_sorted.ms', [
'OBSERVATION_ID', 'ARRAY_ID', 'SCAN_NUMBER', 'FIELD_ID',
'DATA_DESC_ID', 'ANTENNA1', 'ANTENNA2', 'TIME'
])
myms.done()
# Compare both tables. Ignore the DATA column and compare it in next line
self.assertTrue(
th.compTables(
'ms_sorted.ms', 'mms_sorted.ms',
['FLAG_CATEGORY', 'FLAG', 'WEIGHT_SPECTRUM', 'DATA']))
# Compare the DATA column
self.assertTrue(
th.compVarColTables('ms_sorted.ms', 'mms_sorted.ms', 'DATA'))
def test_default_scan(self):
'''Partition: create an MMS with default values and axis=scan'''
partition(vis=self.msfile,
outputvis=self.mmsfile,
separationaxis='scan')
self.assertTrue(os.path.exists(self.mmsfile),
'MMS was not created for this test')
# Take the dictionary and compare with original MS
thisdict = listpartition(vis=self.mmsfile, createdict=True)
# Compare nrows of all scans
slist = ph.getMMSScans(thisdict)
for s in slist:
mmsN = ph.getMMSScanNrows(thisdict, s)
msN = ph.getScanNrows(self.msfile, s)
self.assertEqual(
mmsN, msN,
'Nrows in scan=%s differs: mms_nrows=%s <--> ms_nrows=%s' %
(s, mmsN, msN))
# Compare spw IDs
for s in slist:
mms_spw = ph.getSpwIds(self.mmsfile, s)
ms_spw = ph.getSpwIds(self.msfile, s)
self.assertEqual(mms_spw, ms_spw, 'list of spws in scan=%s differs: '\
'mms_spw=%s <--> ms_spw=%s' %(s, mmsN, msN))
# Sort the output MSs so that they can be compared
myms = mstool()
myms.open(self.msfile)
myms.sort('ms_sorted.ms', [
'OBSERVATION_ID', 'ARRAY_ID', 'SCAN_NUMBER', 'FIELD_ID',
'DATA_DESC_ID', 'ANTENNA1', 'ANTENNA2', 'TIME'
])
myms.done()
myms.open(self.mmsfile)
myms.sort('mms_sorted.ms', [
'OBSERVATION_ID', 'ARRAY_ID', 'SCAN_NUMBER', 'FIELD_ID',
'DATA_DESC_ID', 'ANTENNA1', 'ANTENNA2', 'TIME'
])
myms.done()
self.assertTrue(
th.compTables('ms_sorted.ms', 'mms_sorted.ms', [
'FLAG', 'FLAG_CATEGORY', 'TIME_CENTROID', 'WEIGHT_SPECTRUM',
'DATA'
]))
# Compare the DATA column
self.assertTrue(
th.compVarColTables('ms_sorted.ms', 'mms_sorted.ms', 'DATA'))
def test_nomms(self):
'''Partition: Create a normal MS with createmms=False'''
partition(vis=self.msfile, outputvis=self.mmsfile, createmms=False)
time.sleep(10)
self.assertTrue(os.path.exists(self.mmsfile), 'MMS was not created for this test')
# Compare both tables. Ignore the DATA column and compare it in next line
self.assertTrue(th.compTables(self.msfile, self.mmsfile,
['FLAG_CATEGORY','FLAG','WEIGHT_SPECTRUM','DATA']))
# Compare the DATA column
self.assertTrue(th.compVarColTables(self.msfile,self.mmsfile,'DATA'))
def test_default_scan(self):
"""Partition: create an MMS with default values and axis=scan"""
partition(vis=self.msfile, outputvis=self.mmsfile, separationaxis="scan")
self.assertTrue(os.path.exists(self.mmsfile), "MMS was not created for this test")
# Take the dictionary and compare with original MS
thisdict = listpartition(vis=self.mmsfile, createdict=True)
# Compare nrows of all scans
slist = ph.getMMSScans(thisdict)
for s in slist:
mmsN = ph.getMMSScanNrows(thisdict, s)
msN = ph.getScanNrows(self.msfile, s)
self.assertEqual(mmsN, msN, "Nrows in scan=%s differs: mms_nrows=%s <--> ms_nrows=%s" % (s, mmsN, msN))
# Compare spw IDs
for s in slist:
mms_spw = ph.getSpwIds(self.mmsfile, s)
ms_spw = ph.getSpwIds(self.msfile, s)
self.assertEqual(
mms_spw, ms_spw, "list of spws in scan=%s differs: " "mms_spw=%s <--> ms_spw=%s" % (s, mmsN, msN)
)
# Sort the output MSs so that they can be compared
myms = mstool()
myms.open(self.msfile)
myms.sort(
"ms_sorted.ms",
["OBSERVATION_ID", "ARRAY_ID", "SCAN_NUMBER", "FIELD_ID", "DATA_DESC_ID", "ANTENNA1", "ANTENNA2", "TIME"],
)
myms.done()
myms.open(self.mmsfile)
myms.sort(
"mms_sorted.ms",
["OBSERVATION_ID", "ARRAY_ID", "SCAN_NUMBER", "FIELD_ID", "DATA_DESC_ID", "ANTENNA1", "ANTENNA2", "TIME"],
)
myms.done()
self.assertTrue(
th.compTables(
"ms_sorted.ms", "mms_sorted.ms", ["FLAG", "FLAG_CATEGORY", "TIME_CENTROID", "WEIGHT_SPECTRUM", "DATA"]
)
)
# Compare the DATA column
self.assertTrue(th.compVarColTables("ms_sorted.ms", "mms_sorted.ms", "DATA"))
def test_nomms(self):
'''Partition: Create a normal MS with createmms=False'''
partition(vis=self.msfile, outputvis=self.mmsfile, createmms=False)
time.sleep(10)
self.assertTrue(os.path.exists(self.mmsfile),
'MMS was not created for this test')
# Compare both tables. Ignore the DATA column and compare it in next line
self.assertTrue(
th.compTables(
self.msfile, self.mmsfile,
['FLAG_CATEGORY', 'FLAG', 'WEIGHT_SPECTRUM', 'DATA']))
# Compare the DATA column
self.assertTrue(th.compVarColTables(self.msfile, self.mmsfile, 'DATA'))
def test_default_scan(self):
'''Partition: create an MMS with default values and axis=scan'''
partition(vis=self.msfile, outputvis=self.mmsfile, separationaxis='scan')
self.assertTrue(os.path.exists(self.mmsfile), 'MMS was not created for this test')
# Take the dictionary and compare with original MS
thisdict = listpartition(vis=self.mmsfile, createdict=True)
# Compare nrows of all scans
slist = ph.getMMSScans(thisdict)
for s in slist:
mmsN = ph.getMMSScanNrows(thisdict, s)
msN = ph.getScanNrows(self.msfile, s)
self.assertEqual(mmsN, msN, 'Nrows in scan=%s differs: mms_nrows=%s <--> ms_nrows=%s'
%(s, mmsN, msN))
# Compare spw IDs
for s in slist:
mms_spw = ph.getSpwIds(self.mmsfile, s)
ms_spw = ph.getSpwIds(self.msfile, s)
self.assertEqual(mms_spw, ms_spw, 'list of spws in scan=%s differs: '\
'mms_spw=%s <--> ms_spw=%s' %(s, mmsN, msN))
# Sort the output MSs so that they can be compared
myms = mstool()
myms.open(self.msfile)
myms.sort('ms_sorted.ms',['OBSERVATION_ID','ARRAY_ID','SCAN_NUMBER','FIELD_ID','DATA_DESC_ID','ANTENNA1','ANTENNA2','TIME'])
myms.done()
myms.open(self.mmsfile)
myms.sort('mms_sorted.ms',['OBSERVATION_ID','ARRAY_ID','SCAN_NUMBER','FIELD_ID','DATA_DESC_ID','ANTENNA1','ANTENNA2','TIME'])
myms.done()
self.assertTrue(th.compTables('ms_sorted.ms', 'mms_sorted.ms',
['FLAG','FLAG_CATEGORY','TIME_CENTROID',
'WEIGHT_SPECTRUM','DATA']))
# Compare the DATA column
self.assertTrue(th.compVarColTables('ms_sorted.ms','mms_sorted.ms','DATA'))
def test_nomms(self):
'''Partition: Create a normal MS with createmms=False'''
partition(vis=self.msfile, outputvis=self.mmsfile, createmms=False)
self.assertTrue(os.path.exists(self.mmsfile), 'MMS was not created for this test')
# Sort the output MSs so that they can be compared
myms = mstool()
myms.open(self.msfile)
myms.sort('ms_sorted.ms',['OBSERVATION_ID','ARRAY_ID','SCAN_NUMBER','FIELD_ID','DATA_DESC_ID','ANTENNA1','ANTENNA2','TIME'])
myms.done()
myms.open(self.mmsfile)
myms.sort('mms_sorted.ms',['OBSERVATION_ID','ARRAY_ID','SCAN_NUMBER','FIELD_ID','DATA_DESC_ID','ANTENNA1','ANTENNA2','TIME'])
myms.done()
# Compare both tables. Ignore the DATA column and compare it in next line
self.assertTrue(th.compTables('ms_sorted.ms','mms_sorted.ms',
['FLAG_CATEGORY','FLAG','WEIGHT_SPECTRUM','DATA']))
# Compare the DATA column
self.assertTrue(th.compVarColTables('ms_sorted.ms','mms_sorted.ms','DATA'))
def test_mms7(self):
'''Asdm-import: Test good 12 m ASDM with mixed pol/channelisation input with default filler in lazy mode with reading the BDF flags. Output MMS'''
retValue = {'success': True, 'msgs': "", 'error_msgs': '' }
myasdmname = 'uid___A002_X71e4ae_X317_short'
themsname = myasdmname+".ms"
self.res = importasdm(myasdmname, vis=themsname, lazy=True, bdfflags=True, createmms=True)
self.assertEqual(self.res, None)
print myname, ": Success! Now checking output ..."
mscomponents = set(["ANTENNA/table.dat",
"DATA_DESCRIPTION/table.dat",
"FEED/table.dat",
"FIELD/table.dat",
"FLAG_CMD/table.dat",
"HISTORY/table.dat",
"OBSERVATION/table.dat",
"POINTING/table.dat",
"POLARIZATION/table.dat",
"PROCESSOR/table.dat",
"SOURCE/table.dat",
"SPECTRAL_WINDOW/table.dat",
"STATE/table.dat",
"SYSCAL/table.dat",
"ANTENNA/table.f0",
"DATA_DESCRIPTION/table.f0",
"FEED/table.f0",
"FIELD/table.f0",
"FLAG_CMD/table.f0",
"HISTORY/table.f0",
"OBSERVATION/table.f0",
"POINTING/table.f0",
"POLARIZATION/table.f0",
"PROCESSOR/table.f0",
"SOURCE/table.f0",
"SPECTRAL_WINDOW/table.f0",
"STATE/table.f0",
"SYSCAL/table.f0"
])
for name in mscomponents:
if not os.access(themsname+"/"+name, os.F_OK):
print myname, ": Error ", themsname+"/"+name, "doesn't exist ..."
retValue['success']=False
retValue['error_msgs']=retValue['error_msgs']+themsname+'/'+name+' does not exist'
else:
print myname, ": ", name, "present."
print myname, ": MS exists. All tables present. Try opening as MS ..."
try:
mslocal.open(themsname)
except:
print myname, ": Error Cannot open MS table", themsname
retValue['success']=False
retValue['error_msgs']=retValue['error_msgs']+'Cannot open MS table '+themsname
else:
mslocal.close()
print myname, ": OK. Checking tables in detail ..."
importasdm(asdm=myasdmname, vis='reference.ms', lazy=True, overwrite=True, bdfflags=False, createmms=True)
if(os.path.exists('reference.ms')):
retValue['success'] = th.checkwithtaql("select from [select from reference.ms orderby TIME, DATA_DESC_ID, ANTENNA1, ANTENNA2 ] t1, [select from "
+themsname+" orderby TIME, DATA_DESC_ID, ANTENNA1, ANTENNA2 ] t2 where (not all(near(t1.DATA,t2.DATA, 1.e-06)))") == 0
if not retValue['success']:
print "ERROR: DATA does not agree with reference."
else:
print "DATA columns agree."
retValueTmp = th.checkwithtaql("select from [select from reference.ms orderby TIME, DATA_DESC_ID, ANTENNA1, ANTENNA2 ] t1, [select from "
+themsname+" orderby TIME, DATA_DESC_ID, ANTENNA1, ANTENNA2 ] t2 where (not all(t1.FLAG==t2.FLAG)) ") != 0
if not retValueTmp:
print "ERROR: FLAG columns do agree with reference but they shouldn't."
else:
print "FLAG columns do not agree as expected."
retValue['success'] = retValue['success'] and retValueTmp
for subtname in ["ANTENNA",
"DATA_DESCRIPTION",
"FEED",
"FIELD",
"FLAG_CMD",
"OBSERVATION",
"POLARIZATION",
"PROCESSOR",
"SOURCE",
"SPECTRAL_WINDOW",
"STATE",
"SYSCAL"]:
print "\n*** Subtable ",subtname
excllist = []
if subtname=='SOURCE':
excllist=['POSITION', 'TRANSITION', 'REST_FREQUENCY', 'SYSVEL']
if subtname=='SYSCAL':
excllist=['TANT_SPECTRUM', 'TANT_TSYS_SPECTRUM']
if subtname=='SPECTRAL_WINDOW':
excllist=['CHAN_FREQ', 'CHAN_WIDTH', 'EFFECTIVE_BW', 'RESOLUTION', 'ASSOC_SPW_ID', 'ASSOC_NATURE']
for colname in excllist:
if colname!='ASSOC_NATURE':
retValue['success'] = th.compVarColTables('reference.ms/SPECTRAL_WINDOW',
themsname+'/SPECTRAL_WINDOW', colname, 0.01) and retValue['success']
if subtname=='POLARIZATION':
excllist=['CORR_TYPE', 'CORR_PRODUCT']
for colname in excllist:
retValue['success'] = th.compVarColTables('reference.ms/POLARIZATION',
themsname+'/POLARIZATION', colname, 0.01) and retValue['success']
try:
retValue['success'] = th.compTables('reference.ms/'+subtname,
themsname+'/'+subtname, excllist,
0.01) and retValue['success']
except:
retValue['success'] = False
print "ERROR for table ", subtname
self.assertTrue(retValue['success'],retValue['error_msgs'])
def test_mms6(self):
'''test_mms6: Create MMS and lazy=True'''
retValue = {'success': True, 'msgs': "", 'error_msgs': '' }
myasdmname = 'uid___A002_X71e4ae_X317_short'
themsname = myasdmname+".ms"
flagfile = myasdmname+'_cmd.txt'
self.res = importasdm(myasdmname, vis=themsname, lazy=True, scans='0:1~4', createmms=True,
savecmds=True) # only the first 4 scans to save time
self.assertEqual(self.res, None)
self.assertTrue(ParallelDataHelper.isParallelMS(themsname), 'Output is not a Multi-MS')
self.assertTrue(os.path.exists(flagfile))
print myname, ": Success! Now checking output ..."
mscomponents = set(["ANTENNA/table.dat",
"DATA_DESCRIPTION/table.dat",
"FEED/table.dat",
"FIELD/table.dat",
"FLAG_CMD/table.dat",
"HISTORY/table.dat",
"OBSERVATION/table.dat",
"POINTING/table.dat",
"POLARIZATION/table.dat",
"PROCESSOR/table.dat",
"SOURCE/table.dat",
"SPECTRAL_WINDOW/table.dat",
"STATE/table.dat",
"SYSCAL/table.dat",
"ANTENNA/table.f0",
"DATA_DESCRIPTION/table.f0",
"FEED/table.f0",
"FIELD/table.f0",
"FLAG_CMD/table.f0",
"HISTORY/table.f0",
"OBSERVATION/table.f0",
"POINTING/table.f0",
"POLARIZATION/table.f0",
"PROCESSOR/table.f0",
"SOURCE/table.f0",
"SPECTRAL_WINDOW/table.f0",
"STATE/table.f0",
"SYSCAL/table.f0"
])
for name in mscomponents:
if not os.access(themsname+"/"+name, os.F_OK):
print myname, ": Error ", themsname+"/"+name, "doesn't exist ..."
retValue['success']=False
retValue['error_msgs']=retValue['error_msgs']+themsname+'/'+name+' does not exist'
else:
print myname, ": ", name, "present."
print myname, ": MS exists. All tables present. Try opening as MS ..."
try:
mslocal.open(themsname)
mslocal.close()
print myname, ": MS can be opened. Now testing the changing of the asdmref ..."
mslocal.open(themsname)
mslocal.asdmref("./moved_"+myasdmname)
mslocal.close()
os.system("mv "+myasdmname+" moved_"+myasdmname)
mslocal.open(themsname)
except:
print myname, ": Error Cannot open MS table", themsname
retValue['success']=False
retValue['error_msgs']=retValue['error_msgs']+'Cannot open MS table '+themsname
else:
mslocal.close()
print myname, ": OK. Checking tables in detail ..."
importasdm(asdm="moved_"+myasdmname, vis='reference.ms', lazy=False, overwrite=True, scans='0:1~3', createmms=True)
if(os.path.exists('reference.ms')):
retValue['success'] = th.checkwithtaql("select from [select from reference.ms orderby TIME, DATA_DESC_ID, ANTENNA1, ANTENNA2 ] t1, [select from "
+themsname+" orderby TIME, DATA_DESC_ID, ANTENNA1, ANTENNA2 ] t2 where (not all(near(t1.DATA,t2.DATA, 1.e-06)))") == 0
if not retValue['success']:
print "ERROR: DATA does not agree with reference."
else:
print "DATA columns agree."
retValueTmp = th.checkwithtaql("select from [select from reference.ms orderby TIME, DATA_DESC_ID, ANTENNA1, ANTENNA2 ] t1, [select from "
+themsname+" orderby TIME, DATA_DESC_ID, ANTENNA1, ANTENNA2 ] t2 where (not all(near(t1.WEIGHT,t2.WEIGHT, 1.e-06)))") == 0
if not retValueTmp:
print "ERROR: WEIGHT does not agree with reference."
else:
print "WEIGHT columns agree."
retValueTmp2 = th.checkwithtaql("select from [select from reference.ms orderby TIME, DATA_DESC_ID, ANTENNA1, ANTENNA2 ] t1, [select from "
+themsname+" orderby TIME, DATA_DESC_ID, ANTENNA1, ANTENNA2 ] t2 where (not all(t1.FLAG==t2.FLAG)) ") == 0
if not retValueTmp2:
print "ERROR: FLAG does not agree with reference."
else:
print "FLAG columns agree."
retValue['success'] = retValue['success'] and retValueTmp and retValueTmp2
for subtname in ["ANTENNA",
"DATA_DESCRIPTION",
"FEED",
"FIELD",
"FLAG_CMD",
"OBSERVATION",
"POLARIZATION",
"PROCESSOR",
"SOURCE",
"SPECTRAL_WINDOW",
"STATE",
"SYSCAL"]:
print "\n*** Subtable ",subtname
excllist = []
if subtname=='SOURCE':
excllist=['POSITION', 'TRANSITION', 'REST_FREQUENCY', 'SYSVEL']
if subtname=='SYSCAL':
excllist=['TANT_SPECTRUM', 'TANT_TSYS_SPECTRUM']
if subtname=='SPECTRAL_WINDOW':
excllist=['CHAN_FREQ', 'CHAN_WIDTH', 'EFFECTIVE_BW', 'RESOLUTION', 'ASSOC_SPW_ID', 'ASSOC_NATURE']
for colname in excllist:
if colname!='ASSOC_NATURE':
retValue['success'] = th.compVarColTables('reference.ms/SPECTRAL_WINDOW',
themsname+'/SPECTRAL_WINDOW', colname, 0.01) and retValue['success']
if subtname=='POLARIZATION':
excllist=['CORR_TYPE', 'CORR_PRODUCT']
for colname in excllist:
retValue['success'] = th.compVarColTables('reference.ms/POLARIZATION',
themsname+'/POLARIZATION', colname, 0.01) and retValue['success']
try:
retValue['success'] = th.compTables('reference.ms/'+subtname,
themsname+'/'+subtname, excllist,
0.01) and retValue['success']
except:
retValue['success'] = False
print "ERROR for table ", subtname
os.system("mv moved_"+myasdmname+" "+myasdmname)
self.assertTrue(retValue['success'],retValue['error_msgs'])
def test_mms4(self):
'''test_mms4: Create MMS, lazy=True, with separationaxis=scan and scans selection in ASDM'''
retValue = {'success': True, 'msgs': "", 'error_msgs': '' }
myasdmname = 'uid___A002_X72bc38_X000'
themsname = myasdmname + ".ms"
# only the first 3 scans to save time
importasdm(myasdmname, vis=themsname, lazy=True, scans='0:1~3', createmms=True, separationaxis='scan',
process_flags=False)
self.assertTrue(ParallelDataHelper.isParallelMS(themsname), 'Output is not a Multi-MS')
self.assertEqual(ph.axisType(themsname), 'scan', 'Separation axis of MMS should be scan')
print myname, ": Success! Now checking output ..."
mscomponents = set(["ANTENNA/table.dat",
"DATA_DESCRIPTION/table.dat",
"FEED/table.dat",
"FIELD/table.dat",
"FLAG_CMD/table.dat",
"HISTORY/table.dat",
"OBSERVATION/table.dat",
"POINTING/table.dat",
"POLARIZATION/table.dat",
"PROCESSOR/table.dat",
"SOURCE/table.dat",
"SPECTRAL_WINDOW/table.dat",
"STATE/table.dat",
"SYSCAL/table.dat",
"ANTENNA/table.f0",
"DATA_DESCRIPTION/table.f0",
"FEED/table.f0",
"FIELD/table.f0",
"FLAG_CMD/table.f0",
"HISTORY/table.f0",
"OBSERVATION/table.f0",
"POINTING/table.f0",
"POLARIZATION/table.f0",
"PROCESSOR/table.f0",
"SOURCE/table.f0",
"SPECTRAL_WINDOW/table.f0",
"STATE/table.f0",
"SYSCAL/table.f0"
])
for name in mscomponents:
if not os.access(themsname+"/"+name, os.F_OK):
print myname, ": Error ", themsname+"/"+name, "doesn't exist ..."
retValue['success']=False
retValue['error_msgs']=retValue['error_msgs']+themsname+'/'+name+' does not exist'
else:
print myname, ": ", name, "present."
print myname, ": MS exists. All tables present. Try opening as MS ..."
try:
mslocal.open(themsname)
except:
print myname, ": Error Cannot open MS table", themsname
retValue['success']=False
retValue['error_msgs']=retValue['error_msgs']+'Cannot open MS table '+themsname
else:
mslocal.close()
print myname, ": OK. Checking tables in detail ..."
importasdm(asdm=myasdmname, vis='reference.ms', lazy=False, overwrite=True, scans='0:1~3',
createmms=True, separationaxis='scan', process_flags=False)
if(os.path.exists('reference.ms')):
retValue['success'] = th.checkwithtaql("select from [select from reference.ms orderby TIME, DATA_DESC_ID, ANTENNA1, ANTENNA2 ] t1, [select from "
+themsname+" orderby TIME, DATA_DESC_ID, ANTENNA1, ANTENNA2 ] t2 where (not all(near(t1.DATA,t2.DATA, 1.e-06)))") == 0
if not retValue['success']:
print "ERROR: DATA does not agree with reference."
else:
print "DATA columns agree."
retValueTmp = th.checkwithtaql("select from [select from reference.ms orderby TIME, DATA_DESC_ID, ANTENNA1, ANTENNA2 ] t1, [select from "
+themsname+" orderby TIME, DATA_DESC_ID, ANTENNA1, ANTENNA2 ] t2 where (not all(t1.FLAG==t2.FLAG)) ") == 0
if not retValueTmp:
print "ERROR: FLAG does not agree with reference."
else:
print "FLAG columns agree."
retValue['success'] = retValue['success'] and retValueTmp
for subtname in ["ANTENNA",
"DATA_DESCRIPTION",
"FEED",
"FIELD",
"FLAG_CMD",
"OBSERVATION",
"POINTING",
"POLARIZATION",
"PROCESSOR",
"SOURCE",
"SPECTRAL_WINDOW",
"STATE",
"SYSCAL"]:
print "\n*** Subtable ",subtname
excllist = []
if subtname=='SOURCE':
excllist=['POSITION', 'TRANSITION', 'REST_FREQUENCY', 'SYSVEL']
if subtname=='SYSCAL':
excllist=['TANT_SPECTRUM', 'TANT_TSYS_SPECTRUM']
if subtname=='SPECTRAL_WINDOW':
excllist=['CHAN_FREQ', 'CHAN_WIDTH', 'EFFECTIVE_BW', 'RESOLUTION']
for colname in excllist:
retValue['success'] = th.compVarColTables('reference.ms/SPECTRAL_WINDOW',
themsname+'/SPECTRAL_WINDOW', colname, 0.01) and retValue['success']
try:
retValue['success'] = th.compTables('reference.ms/'+subtname,
themsname+'/'+subtname, excllist,
0.01) and retValue['success']
except:
retValue['success'] = False
print "ERROR for table ", subtname
print retValue
self.assertTrue(retValue['success'],retValue['error_msgs'])
def test_default(self):
'''Partition: create an MMS with default values in parallel'''
# First split off one scan to run the test faster
split(vis=self.msfile, outputvis='split30.ms', datacolumn='DATA', scan='30')
msfile = 'split30.ms'
partition(vis=msfile, outputvis=self.mmsfile)
self.assertTrue(os.path.exists(self.mmsfile), 'MMS was not created for this test')
# Gather several metadata information
# for the MS
mdlocal1 = msmdtool()
mdlocal1.open(msfile)
ms_rows = mdlocal1.nrows()
ms_nscans = mdlocal1.nscans()
ms_nspws = mdlocal1.nspw()
ms_scans = mdlocal1.scannumbers()
mdlocal1.close()
# for the MMS
mdlocal2 = msmdtool()
mdlocal2.open(self.mmsfile)
mms_rows = mdlocal2.nrows()
mms_nscans = mdlocal2.nscans()
mms_nspws = mdlocal2.nspw()
mms_scans = mdlocal2.scannumbers()
mdlocal2.close()
# Compare the number of rows
self.assertEqual(ms_rows, mms_rows, 'Compare total number of rows in MS and MMS')
self.assertEqual(ms_nscans, mms_nscans, 'Compare number of scans')
self.assertEqual(ms_nspws, mms_nspws, 'Compare number of spws')
# Compare the scans
self.assertEqual(ms_scans.all(), mms_scans.all(), 'Compare all scan IDs')
try:
mdlocal1.open(msfile)
mdlocal2.open(self.mmsfile)
# Compare the spws
for i in ms_scans:
msi = mdlocal1.spwsforscan(i)
mmsi = mdlocal2.spwsforscan(i)
self.assertEqual(msi.all(), mmsi.all(), 'Compare spw Ids for a scan')
finally:
mdlocal1.close()
mdlocal2.close()
# Sort the output MSs so that they can be compared
myms = mstool()
myms.open(msfile)
myms.sort('ms_sorted.ms',['OBSERVATION_ID','ARRAY_ID','SCAN_NUMBER','FIELD_ID','DATA_DESC_ID','ANTENNA1','ANTENNA2','TIME'])
myms.done()
myms.open(self.mmsfile)
myms.sort('mms_sorted.ms',['OBSERVATION_ID','ARRAY_ID','SCAN_NUMBER','FIELD_ID','DATA_DESC_ID','ANTENNA1','ANTENNA2','TIME'])
myms.done()
# Ignore WEIGHT_SPECTRUM and SIGMA_SPECTRUM, which are empty columns
self.assertTrue(th.compTables('ms_sorted.ms', 'mms_sorted.ms',
['FLAG','FLAG_CATEGORY','TIME_CENTROID',
'WEIGHT_SPECTRUM','SIGMA_SPECTRUM','DATA']))
# Compare the DATA column
self.assertTrue(th.compVarColTables('ms_sorted.ms', 'mms_sorted.ms','DATA'))
# The separation axis should be written to the output MMS
sepaxis = ph.axisType(self.mmsfile)
self.assertEqual(sepaxis, 'scan,spw', 'Partition did not write AxisType correctly in MMS')
def test_default(self):
"""Partition: create an MMS with default values in parallel"""
# First split off one scan to run the test faster
split(vis=self.msfile, outputvis="split30.ms", datacolumn="DATA", scan="30")
msfile = "split30.ms"
partition(vis=msfile, outputvis=self.mmsfile)
self.assertTrue(os.path.exists(self.mmsfile), "MMS was not created for this test")
# Gather several metadata information
# for the MS
mdlocal1 = msmdtool()
mdlocal1.open(msfile)
ms_rows = mdlocal1.nrows()
ms_nscans = mdlocal1.nscans()
ms_nspws = mdlocal1.nspw()
ms_scans = mdlocal1.scannumbers()
mdlocal1.close()
# for the MMS
mdlocal2 = msmdtool()
mdlocal2.open(self.mmsfile)
mms_rows = mdlocal2.nrows()
mms_nscans = mdlocal2.nscans()
mms_nspws = mdlocal2.nspw()
mms_scans = mdlocal2.scannumbers()
mdlocal2.close()
# Compare the number of rows
self.assertEqual(ms_rows, mms_rows, "Compare total number of rows in MS and MMS")
self.assertEqual(ms_nscans, mms_nscans, "Compare number of scans")
self.assertEqual(ms_nspws, mms_nspws, "Compare number of spws")
# Compare the scans
self.assertEqual(ms_scans.all(), mms_scans.all(), "Compare all scan IDs")
try:
mdlocal1.open(msfile)
mdlocal2.open(self.mmsfile)
# Compare the spws
for i in ms_scans:
msi = mdlocal1.spwsforscan(i)
mmsi = mdlocal2.spwsforscan(i)
self.assertEqual(msi.all(), mmsi.all(), "Compare spw Ids for a scan")
finally:
mdlocal1.close()
mdlocal2.close()
# Sort the output MSs so that they can be compared
myms = mstool()
myms.open(msfile)
myms.sort(
"ms_sorted.ms",
["OBSERVATION_ID", "ARRAY_ID", "SCAN_NUMBER", "FIELD_ID", "DATA_DESC_ID", "ANTENNA1", "ANTENNA2", "TIME"],
)
myms.done()
myms.open(self.mmsfile)
myms.sort(
"mms_sorted.ms",
["OBSERVATION_ID", "ARRAY_ID", "SCAN_NUMBER", "FIELD_ID", "DATA_DESC_ID", "ANTENNA1", "ANTENNA2", "TIME"],
)
myms.done()
self.assertTrue(
th.compTables(
"ms_sorted.ms", "mms_sorted.ms", ["FLAG", "FLAG_CATEGORY", "TIME_CENTROID", "WEIGHT_SPECTRUM", "DATA"]
)
)
# Compare the DATA column
self.assertTrue(th.compVarColTables("ms_sorted.ms", "mms_sorted.ms", "DATA"))
# The separation axis should be written to the output MMS
sepaxis = ph.axisType(self.mmsfile)
self.assertEqual(sepaxis, "scan,spw", "Partition did not write AxisType correctly in MMS")
def test_default(self):
'''Partition: create an MMS with default values in parallel'''
# First split off one scan to run the test faster
split(vis=self.msfile,
outputvis='split30.ms',
datacolumn='DATA',
scan='30')
msfile = 'split30.ms'
partition(vis=msfile, outputvis=self.mmsfile)
self.assertTrue(os.path.exists(self.mmsfile),
'MMS was not created for this test')
# Gather several metadata information
# for the MS
mdlocal1 = msmdtool()
mdlocal1.open(msfile)
ms_rows = mdlocal1.nrows()
ms_nscans = mdlocal1.nscans()
ms_nspws = mdlocal1.nspw()
ms_scans = mdlocal1.scannumbers()
mdlocal1.close()
# for the MMS
mdlocal2 = msmdtool()
mdlocal2.open(self.mmsfile)
mms_rows = mdlocal2.nrows()
mms_nscans = mdlocal2.nscans()
mms_nspws = mdlocal2.nspw()
mms_scans = mdlocal2.scannumbers()
mdlocal2.close()
# Compare the number of rows
self.assertEqual(ms_rows, mms_rows,
'Compare total number of rows in MS and MMS')
self.assertEqual(ms_nscans, mms_nscans, 'Compare number of scans')
self.assertEqual(ms_nspws, mms_nspws, 'Compare number of spws')
# Compare the scans
self.assertEqual(ms_scans.all(), mms_scans.all(),
'Compare all scan IDs')
try:
mdlocal1.open(msfile)
mdlocal2.open(self.mmsfile)
# Compare the spws
for i in ms_scans:
msi = mdlocal1.spwsforscan(i)
mmsi = mdlocal2.spwsforscan(i)
self.assertEqual(msi.all(), mmsi.all(),
'Compare spw Ids for a scan')
finally:
mdlocal1.close()
mdlocal2.close()
# Sort the output MSs so that they can be compared
myms = mstool()
myms.open(msfile)
myms.sort('ms_sorted.ms', [
'OBSERVATION_ID', 'ARRAY_ID', 'SCAN_NUMBER', 'FIELD_ID',
'DATA_DESC_ID', 'ANTENNA1', 'ANTENNA2', 'TIME'
])
myms.done()
myms.open(self.mmsfile)
myms.sort('mms_sorted.ms', [
'OBSERVATION_ID', 'ARRAY_ID', 'SCAN_NUMBER', 'FIELD_ID',
'DATA_DESC_ID', 'ANTENNA1', 'ANTENNA2', 'TIME'
])
myms.done()
# Ignore WEIGHT_SPECTRUM and SIGMA_SPECTRUM, which are empty columns
self.assertTrue(
th.compTables('ms_sorted.ms', 'mms_sorted.ms', [
'FLAG', 'FLAG_CATEGORY', 'TIME_CENTROID', 'WEIGHT_SPECTRUM',
'SIGMA_SPECTRUM', 'DATA'
]))
# Compare the DATA column
self.assertTrue(
th.compVarColTables('ms_sorted.ms', 'mms_sorted.ms', 'DATA'))
# The separation axis should be written to the output MMS
sepaxis = ph.axisType(self.mmsfile)
self.assertEqual(sepaxis, 'scan,spw',
'Partition did not write AxisType correctly in MMS')
def test_mms7(self):
'''Asdm-import: Test good 12 m ASDM with mixed pol/channelisation input with default filler in lazy mode with reading the BDF flags. Output MMS'''
retValue = {'success': True, 'msgs': "", 'error_msgs': ''}
myasdmname = 'uid___A002_X71e4ae_X317_short'
themsname = myasdmname + ".ms"
self.res = importasdm(myasdmname,
vis=themsname,
lazy=True,
bdfflags=True,
createmms=True)
self.assertEqual(self.res, None)
print myname, ": Success! Now checking output ..."
mscomponents = set([
"ANTENNA/table.dat", "DATA_DESCRIPTION/table.dat",
"FEED/table.dat", "FIELD/table.dat", "FLAG_CMD/table.dat",
"HISTORY/table.dat", "OBSERVATION/table.dat", "POINTING/table.dat",
"POLARIZATION/table.dat", "PROCESSOR/table.dat",
"SOURCE/table.dat", "SPECTRAL_WINDOW/table.dat", "STATE/table.dat",
"SYSCAL/table.dat", "ANTENNA/table.f0",
"DATA_DESCRIPTION/table.f0", "FEED/table.f0", "FIELD/table.f0",
"FLAG_CMD/table.f0", "HISTORY/table.f0", "OBSERVATION/table.f0",
"POINTING/table.f0", "POLARIZATION/table.f0", "PROCESSOR/table.f0",
"SOURCE/table.f0", "SPECTRAL_WINDOW/table.f0", "STATE/table.f0",
"SYSCAL/table.f0"
])
for name in mscomponents:
if not os.access(themsname + "/" + name, os.F_OK):
print myname, ": Error ", themsname + "/" + name, "doesn't exist ..."
retValue['success'] = False
retValue['error_msgs'] = retValue[
'error_msgs'] + themsname + '/' + name + ' does not exist'
else:
print myname, ": ", name, "present."
print myname, ": MS exists. All tables present. Try opening as MS ..."
try:
mslocal.open(themsname)
except:
print myname, ": Error Cannot open MS table", themsname
retValue['success'] = False
retValue['error_msgs'] = retValue[
'error_msgs'] + 'Cannot open MS table ' + themsname
else:
mslocal.close()
print myname, ": OK. Checking tables in detail ..."
importasdm(asdm=myasdmname,
vis='reference.ms',
lazy=True,
overwrite=True,
bdfflags=False,
createmms=True)
if (os.path.exists('reference.ms')):
retValue['success'] = th.checkwithtaql(
"select from [select from reference.ms orderby TIME, DATA_DESC_ID, ANTENNA1, ANTENNA2 ] t1, [select from "
+ themsname +
" orderby TIME, DATA_DESC_ID, ANTENNA1, ANTENNA2 ] t2 where (not all(near(t1.DATA,t2.DATA, 1.e-06)))"
) == 0
if not retValue['success']:
print "ERROR: DATA does not agree with reference."
else:
print "DATA columns agree."
retValueTmp = th.checkwithtaql(
"select from [select from reference.ms orderby TIME, DATA_DESC_ID, ANTENNA1, ANTENNA2 ] t1, [select from "
+ themsname +
" orderby TIME, DATA_DESC_ID, ANTENNA1, ANTENNA2 ] t2 where (not all(t1.FLAG==t2.FLAG)) "
) != 0
if not retValueTmp:
print "ERROR: FLAG columns do agree with reference but they shouldn't."
else:
print "FLAG columns do not agree as expected."
retValue['success'] = retValue['success'] and retValueTmp
for subtname in [
"ANTENNA", "DATA_DESCRIPTION", "FEED", "FIELD",
"FLAG_CMD", "OBSERVATION", "POLARIZATION", "PROCESSOR",
"SOURCE", "SPECTRAL_WINDOW", "STATE", "SYSCAL"
]:
print "\n*** Subtable ", subtname
excllist = []
if subtname == 'SOURCE':
excllist = [
'POSITION', 'TRANSITION', 'REST_FREQUENCY',
'SYSVEL'
]
if subtname == 'SYSCAL':
excllist = ['TANT_SPECTRUM', 'TANT_TSYS_SPECTRUM']
if subtname == 'SPECTRAL_WINDOW':
excllist = [
'CHAN_FREQ', 'CHAN_WIDTH', 'EFFECTIVE_BW',
'RESOLUTION', 'ASSOC_SPW_ID', 'ASSOC_NATURE'
]
for colname in excllist:
if colname != 'ASSOC_NATURE':
retValue['success'] = th.compVarColTables(
'reference.ms/SPECTRAL_WINDOW',
themsname + '/SPECTRAL_WINDOW', colname,
0.01) and retValue['success']
if subtname == 'POLARIZATION':
excllist = ['CORR_TYPE', 'CORR_PRODUCT']
for colname in excllist:
retValue['success'] = th.compVarColTables(
'reference.ms/POLARIZATION',
themsname + '/POLARIZATION', colname,
0.01) and retValue['success']
try:
retValue['success'] = th.compTables(
'reference.ms/' + subtname, themsname + '/' +
subtname, excllist, 0.01) and retValue['success']
except:
retValue['success'] = False
print "ERROR for table ", subtname
self.assertTrue(retValue['success'], retValue['error_msgs'])
def test_mms6(self):
'''test_mms6: Create MMS and lazy=True'''
retValue = {'success': True, 'msgs': "", 'error_msgs': ''}
myasdmname = 'uid___A002_X71e4ae_X317_short'
themsname = myasdmname + ".ms"
flagfile = myasdmname + '_cmd.txt'
self.res = importasdm(
myasdmname,
vis=themsname,
lazy=True,
scans='0:1~4',
createmms=True,
savecmds=True) # only the first 4 scans to save time
self.assertEqual(self.res, None)
self.assertTrue(ParallelDataHelper.isParallelMS(themsname),
'Output is not a Multi-MS')
self.assertTrue(os.path.exists(flagfile))
print myname, ": Success! Now checking output ..."
mscomponents = set([
"ANTENNA/table.dat", "DATA_DESCRIPTION/table.dat",
"FEED/table.dat", "FIELD/table.dat", "FLAG_CMD/table.dat",
"HISTORY/table.dat", "OBSERVATION/table.dat", "POINTING/table.dat",
"POLARIZATION/table.dat", "PROCESSOR/table.dat",
"SOURCE/table.dat", "SPECTRAL_WINDOW/table.dat", "STATE/table.dat",
"SYSCAL/table.dat", "ANTENNA/table.f0",
"DATA_DESCRIPTION/table.f0", "FEED/table.f0", "FIELD/table.f0",
"FLAG_CMD/table.f0", "HISTORY/table.f0", "OBSERVATION/table.f0",
"POINTING/table.f0", "POLARIZATION/table.f0", "PROCESSOR/table.f0",
"SOURCE/table.f0", "SPECTRAL_WINDOW/table.f0", "STATE/table.f0",
"SYSCAL/table.f0"
])
for name in mscomponents:
if not os.access(themsname + "/" + name, os.F_OK):
print myname, ": Error ", themsname + "/" + name, "doesn't exist ..."
retValue['success'] = False
retValue['error_msgs'] = retValue[
'error_msgs'] + themsname + '/' + name + ' does not exist'
else:
print myname, ": ", name, "present."
print myname, ": MS exists. All tables present. Try opening as MS ..."
try:
mslocal.open(themsname)
mslocal.close()
print myname, ": MS can be opened. Now testing the changing of the asdmref ..."
mslocal.open(themsname)
mslocal.asdmref("./moved_" + myasdmname)
mslocal.close()
os.system("mv " + myasdmname + " moved_" + myasdmname)
mslocal.open(themsname)
except:
print myname, ": Error Cannot open MS table", themsname
retValue['success'] = False
retValue['error_msgs'] = retValue[
'error_msgs'] + 'Cannot open MS table ' + themsname
else:
mslocal.close()
print myname, ": OK. Checking tables in detail ..."
importasdm(asdm="moved_" + myasdmname,
vis='reference.ms',
lazy=False,
overwrite=True,
scans='0:1~3',
createmms=True)
if (os.path.exists('reference.ms')):
retValue['success'] = th.checkwithtaql(
"select from [select from reference.ms orderby TIME, DATA_DESC_ID, ANTENNA1, ANTENNA2 ] t1, [select from "
+ themsname +
" orderby TIME, DATA_DESC_ID, ANTENNA1, ANTENNA2 ] t2 where (not all(near(t1.DATA,t2.DATA, 1.e-06)))"
) == 0
if not retValue['success']:
print "ERROR: DATA does not agree with reference."
else:
print "DATA columns agree."
retValueTmp = th.checkwithtaql(
"select from [select from reference.ms orderby TIME, DATA_DESC_ID, ANTENNA1, ANTENNA2 ] t1, [select from "
+ themsname +
" orderby TIME, DATA_DESC_ID, ANTENNA1, ANTENNA2 ] t2 where (not all(near(t1.WEIGHT,t2.WEIGHT, 1.e-06)))"
) == 0
if not retValueTmp:
print "ERROR: WEIGHT does not agree with reference."
else:
print "WEIGHT columns agree."
retValueTmp2 = th.checkwithtaql(
"select from [select from reference.ms orderby TIME, DATA_DESC_ID, ANTENNA1, ANTENNA2 ] t1, [select from "
+ themsname +
" orderby TIME, DATA_DESC_ID, ANTENNA1, ANTENNA2 ] t2 where (not all(t1.FLAG==t2.FLAG)) "
) == 0
if not retValueTmp2:
print "ERROR: FLAG does not agree with reference."
else:
print "FLAG columns agree."
retValue['success'] = retValue[
'success'] and retValueTmp and retValueTmp2
for subtname in [
"ANTENNA", "DATA_DESCRIPTION", "FEED", "FIELD",
"FLAG_CMD", "OBSERVATION", "POLARIZATION", "PROCESSOR",
"SOURCE", "SPECTRAL_WINDOW", "STATE", "SYSCAL"
]:
print "\n*** Subtable ", subtname
excllist = []
if subtname == 'SOURCE':
excllist = [
'POSITION', 'TRANSITION', 'REST_FREQUENCY',
'SYSVEL'
]
if subtname == 'SYSCAL':
excllist = ['TANT_SPECTRUM', 'TANT_TSYS_SPECTRUM']
if subtname == 'SPECTRAL_WINDOW':
excllist = [
'CHAN_FREQ', 'CHAN_WIDTH', 'EFFECTIVE_BW',
'RESOLUTION', 'ASSOC_SPW_ID', 'ASSOC_NATURE'
]
for colname in excllist:
if colname != 'ASSOC_NATURE':
retValue['success'] = th.compVarColTables(
'reference.ms/SPECTRAL_WINDOW',
themsname + '/SPECTRAL_WINDOW', colname,
0.01) and retValue['success']
if subtname == 'POLARIZATION':
excllist = ['CORR_TYPE', 'CORR_PRODUCT']
for colname in excllist:
retValue['success'] = th.compVarColTables(
'reference.ms/POLARIZATION',
themsname + '/POLARIZATION', colname,
0.01) and retValue['success']
try:
retValue['success'] = th.compTables(
'reference.ms/' + subtname, themsname + '/' +
subtname, excllist, 0.01) and retValue['success']
except:
retValue['success'] = False
print "ERROR for table ", subtname
os.system("mv moved_" + myasdmname + " " + myasdmname)
self.assertTrue(retValue['success'], retValue['error_msgs'])
def test_mms4(self):
'''test_mms4: Create MMS, lazy=True, with separationaxis=scan and scans selection in ASDM'''
retValue = {'success': True, 'msgs': "", 'error_msgs': ''}
myasdmname = 'uid___A002_X72bc38_X000'
themsname = myasdmname + ".ms"
# only the first 3 scans to save time
importasdm(myasdmname,
vis=themsname,
lazy=True,
scans='0:1~3',
createmms=True,
separationaxis='scan',
process_flags=False)
self.assertTrue(ParallelDataHelper.isParallelMS(themsname),
'Output is not a Multi-MS')
self.assertEqual(ph.axisType(themsname), 'scan',
'Separation axis of MMS should be scan')
print myname, ": Success! Now checking output ..."
mscomponents = set([
"ANTENNA/table.dat", "DATA_DESCRIPTION/table.dat",
"FEED/table.dat", "FIELD/table.dat", "FLAG_CMD/table.dat",
"HISTORY/table.dat", "OBSERVATION/table.dat", "POINTING/table.dat",
"POLARIZATION/table.dat", "PROCESSOR/table.dat",
"SOURCE/table.dat", "SPECTRAL_WINDOW/table.dat", "STATE/table.dat",
"SYSCAL/table.dat", "ANTENNA/table.f0",
"DATA_DESCRIPTION/table.f0", "FEED/table.f0", "FIELD/table.f0",
"FLAG_CMD/table.f0", "HISTORY/table.f0", "OBSERVATION/table.f0",
"POINTING/table.f0", "POLARIZATION/table.f0", "PROCESSOR/table.f0",
"SOURCE/table.f0", "SPECTRAL_WINDOW/table.f0", "STATE/table.f0",
"SYSCAL/table.f0"
])
for name in mscomponents:
if not os.access(themsname + "/" + name, os.F_OK):
print myname, ": Error ", themsname + "/" + name, "doesn't exist ..."
retValue['success'] = False
retValue['error_msgs'] = retValue[
'error_msgs'] + themsname + '/' + name + ' does not exist'
else:
print myname, ": ", name, "present."
print myname, ": MS exists. All tables present. Try opening as MS ..."
try:
mslocal.open(themsname)
except:
print myname, ": Error Cannot open MS table", themsname
retValue['success'] = False
retValue['error_msgs'] = retValue[
'error_msgs'] + 'Cannot open MS table ' + themsname
else:
mslocal.close()
print myname, ": OK. Checking tables in detail ..."
importasdm(asdm=myasdmname,
vis='reference.ms',
lazy=False,
overwrite=True,
scans='0:1~3',
createmms=True,
separationaxis='scan',
process_flags=False)
if (os.path.exists('reference.ms')):
retValue['success'] = th.checkwithtaql(
"select from [select from reference.ms orderby TIME, DATA_DESC_ID, ANTENNA1, ANTENNA2 ] t1, [select from "
+ themsname +
" orderby TIME, DATA_DESC_ID, ANTENNA1, ANTENNA2 ] t2 where (not all(near(t1.DATA,t2.DATA, 1.e-06)))"
) == 0
if not retValue['success']:
print "ERROR: DATA does not agree with reference."
else:
print "DATA columns agree."
retValueTmp = th.checkwithtaql(
"select from [select from reference.ms orderby TIME, DATA_DESC_ID, ANTENNA1, ANTENNA2 ] t1, [select from "
+ themsname +
" orderby TIME, DATA_DESC_ID, ANTENNA1, ANTENNA2 ] t2 where (not all(t1.FLAG==t2.FLAG)) "
) == 0
if not retValueTmp:
print "ERROR: FLAG does not agree with reference."
else:
print "FLAG columns agree."
retValue['success'] = retValue['success'] and retValueTmp
for subtname in [
"ANTENNA", "DATA_DESCRIPTION", "FEED", "FIELD",
"FLAG_CMD", "OBSERVATION", "POINTING", "POLARIZATION",
"PROCESSOR", "SOURCE", "SPECTRAL_WINDOW", "STATE",
"SYSCAL"
]:
print "\n*** Subtable ", subtname
excllist = []
if subtname == 'SOURCE':
excllist = [
'POSITION', 'TRANSITION', 'REST_FREQUENCY',
'SYSVEL'
]
if subtname == 'SYSCAL':
excllist = ['TANT_SPECTRUM', 'TANT_TSYS_SPECTRUM']
if subtname == 'SPECTRAL_WINDOW':
excllist = [
'CHAN_FREQ', 'CHAN_WIDTH', 'EFFECTIVE_BW',
'RESOLUTION'
]
for colname in excllist:
retValue['success'] = th.compVarColTables(
'reference.ms/SPECTRAL_WINDOW',
themsname + '/SPECTRAL_WINDOW', colname,
0.01) and retValue['success']
try:
retValue['success'] = th.compTables(
'reference.ms/' + subtname, themsname + '/' +
subtname, excllist, 0.01) and retValue['success']
except:
retValue['success'] = False
print "ERROR for table ", subtname
print retValue
self.assertTrue(retValue['success'], retValue['error_msgs'])
