def test_mms2(self):
'''test_mms2: Create an MMS with default name and lazy=True'''
myasdmname = 'uid___A002_X71e4ae_X317_short'
themsname = myasdmname+".ms"
importasdm(myasdmname, createmms=True, lazy=True, scans='2')
self.assertTrue(ParallelDataHelper.isParallelMS(themsname), 'Output is not a Multi-MS')
def test_float_data_mms_baseline_auto(self):
'''importasdm: Create an MMS with a FLOAT_DATA column separated per baseline '''
myasdmname = 'uid___A002_X6218fb_X264'
themsname = myasdmname + ".ms"
# The ocorr_mode='ao' option will create a FLOAT_DATA column instead of DATA
importasdm(myasdmname,
vis=themsname,
ocorr_mode='ao',
createmms=True,
scans='1',
separationaxis='baseline',
numsubms=4)
self.assertTrue(ParallelDataHelper.isParallelMS(themsname),
'Output is not a Multi-MS')
self.assertTrue(len(th.getColDesc(themsname, 'FLOAT_DATA')) > 0)
md = msmdtool()
md.open(themsname)
bsl = md.baselines()
md.close()
#diagnoals give the auto-corrs
ac = bsl.diagonal()
self.assertTrue(ac.all(), 'Not all auto-correlations are there')
# Take the dictionary and compare with original MS
thisdict = listpartition(vis=themsname, createdict=True)
self.assertEqual(len(thisdict.keys()), 4,
'There should be 4 subMSs in output MMS')
def test_sd_data_mms(self):
'''importasdm: Create an MMS from a single-dish MS and DATA column '''
myasdmname = 'uid___A002_X6218fb_X264'
themsname = myasdmname+".ms"
importasdm(myasdmname, vis=themsname, scans='1,4', createmms=True, separationaxis='scan', numsubms=2, flagbackup=False)
self.assertTrue(ParallelDataHelper.isParallelMS(themsname), 'Output is not a Multi-MS')
self.assertTrue(len(th.getColDesc(themsname, 'DATA')) > 0)
def test_mms2(self):
'''test_mms2: Create an MMS with default name and lazy=True'''
myasdmname = 'uid___A002_X71e4ae_X317_short'
themsname = myasdmname + ".ms"
importasdm(myasdmname, createmms=True, lazy=True, scans='2')
self.assertTrue(ParallelDataHelper.isParallelMS(themsname),
'Output is not a Multi-MS')
def test_mms3(self):
'''test_mms3: Create MMS with separationaxis=spw and lazy=True'''
myasdmname = 'uid___A002_X71e4ae_X317_short'
themsname = myasdmname+".ms"
importasdm(myasdmname, createmms=True, lazy=True, scans='1,2', separationaxis='spw', flagbackup=False,
process_flags=False)
self.assertTrue(ParallelDataHelper.isParallelMS(themsname), 'Output is not a Multi-MS')
self.assertEqual(ph.axisType(themsname), 'spw', 'Separation axis of MMS should be spw')
def test_float_data_mms(self):
'''importasdm: Create an MMS from a FLOAT_DATA MS '''
myasdmname = 'uid___A002_X6218fb_X264'
themsname = myasdmname+".ms"
# The ocorr_mode='ao' option will create a FLOAT_DATA column instead of DATA
importasdm(myasdmname, vis=themsname, ocorr_mode='ao', createmms=True, scans='1')
self.assertTrue(ParallelDataHelper.isParallelMS(themsname), 'Output is not a Multi-MS')
self.assertTrue(len(th.getColDesc(themsname, 'FLOAT_DATA')) > 0)
def test_online1(self):
'''importasdm: online flags file with spw selection by name'''
myasdmname = 'test_uid___A002_X997a62_X8c-short'
themsname = myasdmname + ".ms"
flagfile = myasdmname + '_cmd.txt'
importasdm(myasdmname, vis=themsname, scans='1', savecmds=True)
flist = open(flagfile, 'r').read().splitlines()
self.assertTrue(flist[0].__contains__('spw'))
self.assertFalse(flist[3].__contains__('WVR#Antenna'))
self.assertFalse(flist[4].__contains__('WVR#Antenna'))
def test_online1(self):
'''importasdm: online flags file with spw selection by name'''
myasdmname = 'test_uid___A002_X997a62_X8c-short'
themsname = myasdmname+".ms"
flagfile = myasdmname+'_cmd.txt'
importasdm(myasdmname, vis=themsname, scans='1', savecmds=True)
flist = open(flagfile,'r').read().splitlines()
self.assertTrue(flist[0].__contains__('spw'))
self.assertFalse(flist[3].__contains__('WVR#Antenna'))
self.assertFalse(flist[4].__contains__('WVR#Antenna'))
def test_float_data_mms(self):
'''importasdm: Create an MMS from a FLOAT_DATA MS '''
myasdmname = 'uid___A002_X6218fb_X264'
themsname = myasdmname + ".ms"
# The ocorr_mode='ao' option will create a FLOAT_DATA column instead of DATA
importasdm(myasdmname,
vis=themsname,
ocorr_mode='ao',
createmms=True,
scans='1')
self.assertTrue(ParallelDataHelper.isParallelMS(themsname),
'Output is not a Multi-MS')
self.assertTrue(len(th.getColDesc(themsname, 'FLOAT_DATA')) > 0)
def test_mms5(self):
'''test_mms5: Create 2 MMS, 2 flagversions and 2 online flag files'''
myasdmname = 'uid___A002_X71e4ae_X317_short'
themsname = myasdmname+".ms"
wvrmsname = myasdmname+'-wvr-corrected.ms'
flagfile1 = myasdmname+'_cmd.txt'
flagfile2 = myasdmname+'-wvr-corrected'+'_cmd.txt'
importasdm(myasdmname, vis=themsname, lazy=True, scans='0:1~4', wvr_corrected_data='both', savecmds=True,
createmms=True)
self.assertTrue(ParallelDataHelper.isParallelMS(themsname), 'Output is not a Multi-MS')
self.assertTrue(ParallelDataHelper.isParallelMS(wvrmsname), 'Output is not a Multi-MS')
self.assertTrue(os.path.exists(flagfile1))
self.assertTrue(os.path.exists(flagfile2))
def test_sd_data_mms(self):
'''importasdm: Create an MMS from a single-dish MS and DATA column '''
myasdmname = 'uid___A002_X6218fb_X264'
themsname = myasdmname + ".ms"
importasdm(myasdmname,
vis=themsname,
scans='1,4',
createmms=True,
separationaxis='scan',
numsubms=2,
flagbackup=False)
self.assertTrue(ParallelDataHelper.isParallelMS(themsname),
'Output is not a Multi-MS')
self.assertTrue(len(th.getColDesc(themsname, 'DATA')) > 0)
def test_mms3(self):
'''test_mms3: Create MMS with separationaxis=spw and lazy=True'''
myasdmname = 'uid___A002_X71e4ae_X317_short'
themsname = myasdmname + ".ms"
importasdm(myasdmname,
createmms=True,
lazy=True,
scans='1,2',
separationaxis='spw',
flagbackup=False,
process_flags=False)
self.assertTrue(ParallelDataHelper.isParallelMS(themsname),
'Output is not a Multi-MS')
self.assertEqual(ph.axisType(themsname), 'spw',
'Separation axis of MMS should be spw')
def test_mms_ephem(self):
'''Asdm-import: Test good 12 m ASDM with Ephemeris table in lazy mode and MMS'''
retValue = {'success': True, 'msgs': "", 'error_msgs': '' }
myasdmname = 'uid___A002_X997a62_X8c-short'
themsname = myasdmname+".ms"
self.res = importasdm(myasdmname, vis=themsname, lazy=True, convert_ephem2geo=True,
process_pointing=False, flagbackup=False, createmms=True)
self.assertEqual(self.res, None)
print myname, ": Success! Now checking output ..."
mscomponents = set(["FIELD/table.dat",
"FIELD/EPHEM0_Mars_57034.9.tab",
"FIELD/EPHEM1_Titania_57034.9.tab"
])
for name in mscomponents:
if not os.access(themsname+"/"+name, os.F_OK):
print myname, ": Error ", themsname+"/"+name, "doesn't exist ..."
retValue['success']=False
retValue['error_msgs']=retValue['error_msgs']+themsname+'/'+name+' does not exist'
else:
print myname, ": ", name, "present."
print myname, ": MS exists. All relevant tables present. Try opening as MS ..."
try:
mslocal.open(themsname)
except:
print myname, ": Error Cannot open MS table", themsname
retValue['success']=False
retValue['error_msgs']=retValue['error_msgs']+'Cannot open MS table '+themsname
else:
mslocal.close()
print myname, ": OK."
for name in ["FIELD/EPHEM0_Mars_57034.9.tab",
"FIELD/EPHEM1_Titania_57034.9.tab"]:
tblocal.open(themsname+"/"+name)
kw = tblocal.getkeywords()
tblocal.close()
geodist = kw['GeoDist'] # (km above reference ellipsoid)
geolat = kw['GeoLat'] # (deg)
geolong = kw['GeoLong'] # (deg)
print myname, ": Testing if ephemeris ", name, " was converted to GEO ..."
if not (geodist==geolat==geolong==0.):
print myname, ": ERROR."
retValue['success']=False
retValue['error_msgs']=retValue['error_msgs']+' Ephemeris was not converted to GEO for '+themsname+'\n'
else:
print myname, ": OK."
prsys = kw['posrefsys']
print myname, ": Testing if posrefsys was set correctly ..."
if not (prsys=="ICRF/ICRS"):
print myname, ": ERROR."
retValue['success']=False
retValue['error_msgs']=retValue['error_msgs']+' posrefsys keyword is not ICRF/ICRS '+themsname+'\n'
else:
print myname, ": OK."
self.assertTrue(retValue['success'],retValue['error_msgs'])
def test_sd_data_mms_baseline_all(self):
'''importasdm: Create an MMS separated per baseline, using default numsubms '''
myasdmname = 'uid___A002_X6218fb_X264'
themsname = myasdmname+".ms"
# Create a single-dish MMS with auto and cross correlations
importasdm(myasdmname, vis=themsname, ocorr_mode='ca', createmms=True, scans='1', separationaxis='baseline')
self.assertTrue(ParallelDataHelper.isParallelMS(themsname), 'Output is not a Multi-MS')
self.assertTrue(len(th.getColDesc(themsname, 'DATA')) > 0)
md = msmdtool()
md.open(themsname)
bsl = md.baselines()
md.close()
# Check if all baselines are in there
self.assertTrue(bsl.all(), 'Not all baselines are in the MMS')
def test_mms5(self):
'''test_mms5: Create 2 MMS, 2 flagversions and 2 online flag files'''
myasdmname = 'uid___A002_X71e4ae_X317_short'
themsname = myasdmname + ".ms"
wvrmsname = myasdmname + '-wvr-corrected.ms'
flagfile1 = myasdmname + '_cmd.txt'
flagfile2 = myasdmname + '-wvr-corrected' + '_cmd.txt'
importasdm(myasdmname,
vis=themsname,
lazy=True,
scans='0:1~4',
wvr_corrected_data='both',
savecmds=True,
createmms=True)
self.assertTrue(ParallelDataHelper.isParallelMS(themsname),
'Output is not a Multi-MS')
self.assertTrue(ParallelDataHelper.isParallelMS(wvrmsname),
'Output is not a Multi-MS')
self.assertTrue(os.path.exists(flagfile1))
self.assertTrue(os.path.exists(flagfile2))
def test_float_data_mms_baseline_auto(self):
'''importasdm: Create an MMS with a FLOAT_DATA column separated per baseline '''
myasdmname = 'uid___A002_X6218fb_X264'
themsname = myasdmname+".ms"
# The ocorr_mode='ao' option will create a FLOAT_DATA column instead of DATA
importasdm(myasdmname, vis=themsname, ocorr_mode='ao', createmms=True, scans='1', separationaxis='baseline', numsubms=4)
self.assertTrue(ParallelDataHelper.isParallelMS(themsname), 'Output is not a Multi-MS')
self.assertTrue(len(th.getColDesc(themsname, 'FLOAT_DATA')) > 0)
md = msmdtool()
md.open(themsname)
bsl = md.baselines()
md.close()
#diagnoals give the auto-corrs
ac = bsl.diagonal()
self.assertTrue(ac.all(), 'Not all auto-correlations are there')
# Take the dictionary and compare with original MS
thisdict = listpartition(vis=themsname, createdict=True)
self.assertEqual(len(thisdict.keys()), 4, 'There should be 4 subMSs in output MMS')
def test_sd_data_mms_baseline_all(self):
'''importasdm: Create an MMS separated per baseline, using default numsubms '''
myasdmname = 'uid___A002_X6218fb_X264'
themsname = myasdmname + ".ms"
# Create a single-dish MMS with auto and cross correlations
importasdm(myasdmname,
vis=themsname,
ocorr_mode='ca',
createmms=True,
scans='1',
separationaxis='baseline')
self.assertTrue(ParallelDataHelper.isParallelMS(themsname),
'Output is not a Multi-MS')
self.assertTrue(len(th.getColDesc(themsname, 'DATA')) > 0)
md = msmdtool()
md.open(themsname)
bsl = md.baselines()
md.close()
# Check if all baselines are in there
self.assertTrue(bsl.all(), 'Not all baselines are in the MMS')
split_lines = True
else:
raise ValueError("Input for splitting should be lines, "
"cont or all. Given {}".format(which_split))
ms_active = mySDM + ".ms"
print("Given inputs:")
print("SDM: {}".format(mySDM))
print("Make MMS: {}".format(parallel_run))
if not os.path.exists(ms_active):
importasdm(asdm=mySDM,
vis=ms_active,
ocorr_mode='co',
applyflags=True,
savecmds=True,
tbuff=1.5,
outfile='{}.flagonline.txt'.format(mySDM),
createmms=parallel_run)
else:
print("MS already exists. Skipping importasdm")
parentdir = os.getcwd().split("/")[-1]
if split_lines:
lines_folder = parentdir + '_speclines'
if not os.path.exists(lines_folder):
os.mkdir(lines_folder)
# Copy the cont.dat file from the repo
def test_mms4(self):
'''test_mms4: Create MMS, lazy=True, with separationaxis=scan and scans selection in ASDM'''
retValue = {'success': True, 'msgs': "", 'error_msgs': ''}
myasdmname = 'uid___A002_X72bc38_X000'
themsname = myasdmname + ".ms"
# only the first 3 scans to save time
importasdm(myasdmname,
vis=themsname,
lazy=True,
scans='0:1~3',
createmms=True,
separationaxis='scan',
process_flags=False)
self.assertTrue(ParallelDataHelper.isParallelMS(themsname),
'Output is not a Multi-MS')
self.assertEqual(ph.axisType(themsname), 'scan',
'Separation axis of MMS should be scan')
print myname, ": Success! Now checking output ..."
mscomponents = set([
"ANTENNA/table.dat", "DATA_DESCRIPTION/table.dat",
"FEED/table.dat", "FIELD/table.dat", "FLAG_CMD/table.dat",
"HISTORY/table.dat", "OBSERVATION/table.dat", "POINTING/table.dat",
"POLARIZATION/table.dat", "PROCESSOR/table.dat",
"SOURCE/table.dat", "SPECTRAL_WINDOW/table.dat", "STATE/table.dat",
"SYSCAL/table.dat", "ANTENNA/table.f0",
"DATA_DESCRIPTION/table.f0", "FEED/table.f0", "FIELD/table.f0",
"FLAG_CMD/table.f0", "HISTORY/table.f0", "OBSERVATION/table.f0",
"POINTING/table.f0", "POLARIZATION/table.f0", "PROCESSOR/table.f0",
"SOURCE/table.f0", "SPECTRAL_WINDOW/table.f0", "STATE/table.f0",
"SYSCAL/table.f0"
])
for name in mscomponents:
if not os.access(themsname + "/" + name, os.F_OK):
print myname, ": Error ", themsname + "/" + name, "doesn't exist ..."
retValue['success'] = False
retValue['error_msgs'] = retValue[
'error_msgs'] + themsname + '/' + name + ' does not exist'
else:
print myname, ": ", name, "present."
print myname, ": MS exists. All tables present. Try opening as MS ..."
try:
mslocal.open(themsname)
except:
print myname, ": Error Cannot open MS table", themsname
retValue['success'] = False
retValue['error_msgs'] = retValue[
'error_msgs'] + 'Cannot open MS table ' + themsname
else:
mslocal.close()
print myname, ": OK. Checking tables in detail ..."
importasdm(asdm=myasdmname,
vis='reference.ms',
lazy=False,
overwrite=True,
scans='0:1~3',
createmms=True,
separationaxis='scan',
process_flags=False)
if (os.path.exists('reference.ms')):
retValue['success'] = th.checkwithtaql(
"select from [select from reference.ms orderby TIME, DATA_DESC_ID, ANTENNA1, ANTENNA2 ] t1, [select from "
+ themsname +
" orderby TIME, DATA_DESC_ID, ANTENNA1, ANTENNA2 ] t2 where (not all(near(t1.DATA,t2.DATA, 1.e-06)))"
) == 0
if not retValue['success']:
print "ERROR: DATA does not agree with reference."
else:
print "DATA columns agree."
retValueTmp = th.checkwithtaql(
"select from [select from reference.ms orderby TIME, DATA_DESC_ID, ANTENNA1, ANTENNA2 ] t1, [select from "
+ themsname +
" orderby TIME, DATA_DESC_ID, ANTENNA1, ANTENNA2 ] t2 where (not all(t1.FLAG==t2.FLAG)) "
) == 0
if not retValueTmp:
print "ERROR: FLAG does not agree with reference."
else:
print "FLAG columns agree."
retValue['success'] = retValue['success'] and retValueTmp
for subtname in [
"ANTENNA", "DATA_DESCRIPTION", "FEED", "FIELD",
"FLAG_CMD", "OBSERVATION", "POINTING", "POLARIZATION",
"PROCESSOR", "SOURCE", "SPECTRAL_WINDOW", "STATE",
"SYSCAL"
]:
print "\n*** Subtable ", subtname
excllist = []
if subtname == 'SOURCE':
excllist = [
'POSITION', 'TRANSITION', 'REST_FREQUENCY',
'SYSVEL'
]
if subtname == 'SYSCAL':
excllist = ['TANT_SPECTRUM', 'TANT_TSYS_SPECTRUM']
if subtname == 'SPECTRAL_WINDOW':
excllist = [
'CHAN_FREQ', 'CHAN_WIDTH', 'EFFECTIVE_BW',
'RESOLUTION'
]
for colname in excllist:
retValue['success'] = th.compVarColTables(
'reference.ms/SPECTRAL_WINDOW',
themsname + '/SPECTRAL_WINDOW', colname,
0.01) and retValue['success']
try:
retValue['success'] = th.compTables(
'reference.ms/' + subtname, themsname + '/' +
subtname, excllist, 0.01) and retValue['success']
except:
retValue['success'] = False
print "ERROR for table ", subtname
print retValue
self.assertTrue(retValue['success'], retValue['error_msgs'])
def test_mms6(self):
'''test_mms6: Create MMS and lazy=True'''
retValue = {'success': True, 'msgs': "", 'error_msgs': ''}
myasdmname = 'uid___A002_X71e4ae_X317_short'
themsname = myasdmname + ".ms"
flagfile = myasdmname + '_cmd.txt'
self.res = importasdm(
myasdmname,
vis=themsname,
lazy=True,
scans='0:1~4',
createmms=True,
savecmds=True) # only the first 4 scans to save time
self.assertEqual(self.res, None)
self.assertTrue(ParallelDataHelper.isParallelMS(themsname),
'Output is not a Multi-MS')
self.assertTrue(os.path.exists(flagfile))
print myname, ": Success! Now checking output ..."
mscomponents = set([
"ANTENNA/table.dat", "DATA_DESCRIPTION/table.dat",
"FEED/table.dat", "FIELD/table.dat", "FLAG_CMD/table.dat",
"HISTORY/table.dat", "OBSERVATION/table.dat", "POINTING/table.dat",
"POLARIZATION/table.dat", "PROCESSOR/table.dat",
"SOURCE/table.dat", "SPECTRAL_WINDOW/table.dat", "STATE/table.dat",
"SYSCAL/table.dat", "ANTENNA/table.f0",
"DATA_DESCRIPTION/table.f0", "FEED/table.f0", "FIELD/table.f0",
"FLAG_CMD/table.f0", "HISTORY/table.f0", "OBSERVATION/table.f0",
"POINTING/table.f0", "POLARIZATION/table.f0", "PROCESSOR/table.f0",
"SOURCE/table.f0", "SPECTRAL_WINDOW/table.f0", "STATE/table.f0",
"SYSCAL/table.f0"
])
for name in mscomponents:
if not os.access(themsname + "/" + name, os.F_OK):
print myname, ": Error ", themsname + "/" + name, "doesn't exist ..."
retValue['success'] = False
retValue['error_msgs'] = retValue[
'error_msgs'] + themsname + '/' + name + ' does not exist'
else:
print myname, ": ", name, "present."
print myname, ": MS exists. All tables present. Try opening as MS ..."
try:
mslocal.open(themsname)
mslocal.close()
print myname, ": MS can be opened. Now testing the changing of the asdmref ..."
mslocal.open(themsname)
mslocal.asdmref("./moved_" + myasdmname)
mslocal.close()
os.system("mv " + myasdmname + " moved_" + myasdmname)
mslocal.open(themsname)
except:
print myname, ": Error Cannot open MS table", themsname
retValue['success'] = False
retValue['error_msgs'] = retValue[
'error_msgs'] + 'Cannot open MS table ' + themsname
else:
mslocal.close()
print myname, ": OK. Checking tables in detail ..."
importasdm(asdm="moved_" + myasdmname,
vis='reference.ms',
lazy=False,
overwrite=True,
scans='0:1~3',
createmms=True)
if (os.path.exists('reference.ms')):
retValue['success'] = th.checkwithtaql(
"select from [select from reference.ms orderby TIME, DATA_DESC_ID, ANTENNA1, ANTENNA2 ] t1, [select from "
+ themsname +
" orderby TIME, DATA_DESC_ID, ANTENNA1, ANTENNA2 ] t2 where (not all(near(t1.DATA,t2.DATA, 1.e-06)))"
) == 0
if not retValue['success']:
print "ERROR: DATA does not agree with reference."
else:
print "DATA columns agree."
retValueTmp = th.checkwithtaql(
"select from [select from reference.ms orderby TIME, DATA_DESC_ID, ANTENNA1, ANTENNA2 ] t1, [select from "
+ themsname +
" orderby TIME, DATA_DESC_ID, ANTENNA1, ANTENNA2 ] t2 where (not all(near(t1.WEIGHT,t2.WEIGHT, 1.e-06)))"
) == 0
if not retValueTmp:
print "ERROR: WEIGHT does not agree with reference."
else:
print "WEIGHT columns agree."
retValueTmp2 = th.checkwithtaql(
"select from [select from reference.ms orderby TIME, DATA_DESC_ID, ANTENNA1, ANTENNA2 ] t1, [select from "
+ themsname +
" orderby TIME, DATA_DESC_ID, ANTENNA1, ANTENNA2 ] t2 where (not all(t1.FLAG==t2.FLAG)) "
) == 0
if not retValueTmp2:
print "ERROR: FLAG does not agree with reference."
else:
print "FLAG columns agree."
retValue['success'] = retValue[
'success'] and retValueTmp and retValueTmp2
for subtname in [
"ANTENNA", "DATA_DESCRIPTION", "FEED", "FIELD",
"FLAG_CMD", "OBSERVATION", "POLARIZATION", "PROCESSOR",
"SOURCE", "SPECTRAL_WINDOW", "STATE", "SYSCAL"
]:
print "\n*** Subtable ", subtname
excllist = []
if subtname == 'SOURCE':
excllist = [
'POSITION', 'TRANSITION', 'REST_FREQUENCY',
'SYSVEL'
]
if subtname == 'SYSCAL':
excllist = ['TANT_SPECTRUM', 'TANT_TSYS_SPECTRUM']
if subtname == 'SPECTRAL_WINDOW':
excllist = [
'CHAN_FREQ', 'CHAN_WIDTH', 'EFFECTIVE_BW',
'RESOLUTION', 'ASSOC_SPW_ID', 'ASSOC_NATURE'
]
for colname in excllist:
if colname != 'ASSOC_NATURE':
retValue['success'] = th.compVarColTables(
'reference.ms/SPECTRAL_WINDOW',
themsname + '/SPECTRAL_WINDOW', colname,
0.01) and retValue['success']
if subtname == 'POLARIZATION':
excllist = ['CORR_TYPE', 'CORR_PRODUCT']
for colname in excllist:
retValue['success'] = th.compVarColTables(
'reference.ms/POLARIZATION',
themsname + '/POLARIZATION', colname,
0.01) and retValue['success']
try:
retValue['success'] = th.compTables(
'reference.ms/' + subtname, themsname + '/' +
subtname, excllist, 0.01) and retValue['success']
except:
retValue['success'] = False
print "ERROR for table ", subtname
os.system("mv moved_" + myasdmname + " " + myasdmname)
self.assertTrue(retValue['success'], retValue['error_msgs'])
def test_mms7(self):
'''Asdm-import: Test good 12 m ASDM with mixed pol/channelisation input with default filler in lazy mode with reading the BDF flags. Output MMS'''
retValue = {'success': True, 'msgs': "", 'error_msgs': '' }
myasdmname = 'uid___A002_X71e4ae_X317_short'
themsname = myasdmname+".ms"
self.res = importasdm(myasdmname, vis=themsname, lazy=True, bdfflags=True, createmms=True)
self.assertEqual(self.res, None)
print myname, ": Success! Now checking output ..."
mscomponents = set(["ANTENNA/table.dat",
"DATA_DESCRIPTION/table.dat",
"FEED/table.dat",
"FIELD/table.dat",
"FLAG_CMD/table.dat",
"HISTORY/table.dat",
"OBSERVATION/table.dat",
"POINTING/table.dat",
"POLARIZATION/table.dat",
"PROCESSOR/table.dat",
"SOURCE/table.dat",
"SPECTRAL_WINDOW/table.dat",
"STATE/table.dat",
"SYSCAL/table.dat",
"ANTENNA/table.f0",
"DATA_DESCRIPTION/table.f0",
"FEED/table.f0",
"FIELD/table.f0",
"FLAG_CMD/table.f0",
"HISTORY/table.f0",
"OBSERVATION/table.f0",
"POINTING/table.f0",
"POLARIZATION/table.f0",
"PROCESSOR/table.f0",
"SOURCE/table.f0",
"SPECTRAL_WINDOW/table.f0",
"STATE/table.f0",
"SYSCAL/table.f0"
])
for name in mscomponents:
if not os.access(themsname+"/"+name, os.F_OK):
print myname, ": Error ", themsname+"/"+name, "doesn't exist ..."
retValue['success']=False
retValue['error_msgs']=retValue['error_msgs']+themsname+'/'+name+' does not exist'
else:
print myname, ": ", name, "present."
print myname, ": MS exists. All tables present. Try opening as MS ..."
try:
mslocal.open(themsname)
except:
print myname, ": Error Cannot open MS table", themsname
retValue['success']=False
retValue['error_msgs']=retValue['error_msgs']+'Cannot open MS table '+themsname
else:
mslocal.close()
print myname, ": OK. Checking tables in detail ..."
importasdm(asdm=myasdmname, vis='reference.ms', lazy=True, overwrite=True, bdfflags=False, createmms=True)
if(os.path.exists('reference.ms')):
retValue['success'] = th.checkwithtaql("select from [select from reference.ms orderby TIME, DATA_DESC_ID, ANTENNA1, ANTENNA2 ] t1, [select from "
+themsname+" orderby TIME, DATA_DESC_ID, ANTENNA1, ANTENNA2 ] t2 where (not all(near(t1.DATA,t2.DATA, 1.e-06)))") == 0
if not retValue['success']:
print "ERROR: DATA does not agree with reference."
else:
print "DATA columns agree."
retValueTmp = th.checkwithtaql("select from [select from reference.ms orderby TIME, DATA_DESC_ID, ANTENNA1, ANTENNA2 ] t1, [select from "
+themsname+" orderby TIME, DATA_DESC_ID, ANTENNA1, ANTENNA2 ] t2 where (not all(t1.FLAG==t2.FLAG)) ") != 0
if not retValueTmp:
print "ERROR: FLAG columns do agree with reference but they shouldn't."
else:
print "FLAG columns do not agree as expected."
retValue['success'] = retValue['success'] and retValueTmp
for subtname in ["ANTENNA",
"DATA_DESCRIPTION",
"FEED",
"FIELD",
"FLAG_CMD",
"OBSERVATION",
"POLARIZATION",
"PROCESSOR",
"SOURCE",
"SPECTRAL_WINDOW",
"STATE",
"SYSCAL"]:
print "\n*** Subtable ",subtname
excllist = []
if subtname=='SOURCE':
excllist=['POSITION', 'TRANSITION', 'REST_FREQUENCY', 'SYSVEL']
if subtname=='SYSCAL':
excllist=['TANT_SPECTRUM', 'TANT_TSYS_SPECTRUM']
if subtname=='SPECTRAL_WINDOW':
excllist=['CHAN_FREQ', 'CHAN_WIDTH', 'EFFECTIVE_BW', 'RESOLUTION', 'ASSOC_SPW_ID', 'ASSOC_NATURE']
for colname in excllist:
if colname!='ASSOC_NATURE':
retValue['success'] = th.compVarColTables('reference.ms/SPECTRAL_WINDOW',
themsname+'/SPECTRAL_WINDOW', colname, 0.01) and retValue['success']
if subtname=='POLARIZATION':
excllist=['CORR_TYPE', 'CORR_PRODUCT']
for colname in excllist:
retValue['success'] = th.compVarColTables('reference.ms/POLARIZATION',
themsname+'/POLARIZATION', colname, 0.01) and retValue['success']
try:
retValue['success'] = th.compTables('reference.ms/'+subtname,
themsname+'/'+subtname, excllist,
0.01) and retValue['success']
except:
retValue['success'] = False
print "ERROR for table ", subtname
self.assertTrue(retValue['success'],retValue['error_msgs'])
def test_mms6(self):
'''test_mms6: Create MMS and lazy=True'''
retValue = {'success': True, 'msgs': "", 'error_msgs': '' }
myasdmname = 'uid___A002_X71e4ae_X317_short'
themsname = myasdmname+".ms"
flagfile = myasdmname+'_cmd.txt'
self.res = importasdm(myasdmname, vis=themsname, lazy=True, scans='0:1~4', createmms=True,
savecmds=True) # only the first 4 scans to save time
self.assertEqual(self.res, None)
self.assertTrue(ParallelDataHelper.isParallelMS(themsname), 'Output is not a Multi-MS')
self.assertTrue(os.path.exists(flagfile))
print myname, ": Success! Now checking output ..."
mscomponents = set(["ANTENNA/table.dat",
"DATA_DESCRIPTION/table.dat",
"FEED/table.dat",
"FIELD/table.dat",
"FLAG_CMD/table.dat",
"HISTORY/table.dat",
"OBSERVATION/table.dat",
"POINTING/table.dat",
"POLARIZATION/table.dat",
"PROCESSOR/table.dat",
"SOURCE/table.dat",
"SPECTRAL_WINDOW/table.dat",
"STATE/table.dat",
"SYSCAL/table.dat",
"ANTENNA/table.f0",
"DATA_DESCRIPTION/table.f0",
"FEED/table.f0",
"FIELD/table.f0",
"FLAG_CMD/table.f0",
"HISTORY/table.f0",
"OBSERVATION/table.f0",
"POINTING/table.f0",
"POLARIZATION/table.f0",
"PROCESSOR/table.f0",
"SOURCE/table.f0",
"SPECTRAL_WINDOW/table.f0",
"STATE/table.f0",
"SYSCAL/table.f0"
])
for name in mscomponents:
if not os.access(themsname+"/"+name, os.F_OK):
print myname, ": Error ", themsname+"/"+name, "doesn't exist ..."
retValue['success']=False
retValue['error_msgs']=retValue['error_msgs']+themsname+'/'+name+' does not exist'
else:
print myname, ": ", name, "present."
print myname, ": MS exists. All tables present. Try opening as MS ..."
try:
mslocal.open(themsname)
mslocal.close()
print myname, ": MS can be opened. Now testing the changing of the asdmref ..."
mslocal.open(themsname)
mslocal.asdmref("./moved_"+myasdmname)
mslocal.close()
os.system("mv "+myasdmname+" moved_"+myasdmname)
mslocal.open(themsname)
except:
print myname, ": Error Cannot open MS table", themsname
retValue['success']=False
retValue['error_msgs']=retValue['error_msgs']+'Cannot open MS table '+themsname
else:
mslocal.close()
print myname, ": OK. Checking tables in detail ..."
importasdm(asdm="moved_"+myasdmname, vis='reference.ms', lazy=False, overwrite=True, scans='0:1~3', createmms=True)
if(os.path.exists('reference.ms')):
retValue['success'] = th.checkwithtaql("select from [select from reference.ms orderby TIME, DATA_DESC_ID, ANTENNA1, ANTENNA2 ] t1, [select from "
+themsname+" orderby TIME, DATA_DESC_ID, ANTENNA1, ANTENNA2 ] t2 where (not all(near(t1.DATA,t2.DATA, 1.e-06)))") == 0
if not retValue['success']:
print "ERROR: DATA does not agree with reference."
else:
print "DATA columns agree."
retValueTmp = th.checkwithtaql("select from [select from reference.ms orderby TIME, DATA_DESC_ID, ANTENNA1, ANTENNA2 ] t1, [select from "
+themsname+" orderby TIME, DATA_DESC_ID, ANTENNA1, ANTENNA2 ] t2 where (not all(near(t1.WEIGHT,t2.WEIGHT, 1.e-06)))") == 0
if not retValueTmp:
print "ERROR: WEIGHT does not agree with reference."
else:
print "WEIGHT columns agree."
retValueTmp2 = th.checkwithtaql("select from [select from reference.ms orderby TIME, DATA_DESC_ID, ANTENNA1, ANTENNA2 ] t1, [select from "
+themsname+" orderby TIME, DATA_DESC_ID, ANTENNA1, ANTENNA2 ] t2 where (not all(t1.FLAG==t2.FLAG)) ") == 0
if not retValueTmp2:
print "ERROR: FLAG does not agree with reference."
else:
print "FLAG columns agree."
retValue['success'] = retValue['success'] and retValueTmp and retValueTmp2
for subtname in ["ANTENNA",
"DATA_DESCRIPTION",
"FEED",
"FIELD",
"FLAG_CMD",
"OBSERVATION",
"POLARIZATION",
"PROCESSOR",
"SOURCE",
"SPECTRAL_WINDOW",
"STATE",
"SYSCAL"]:
print "\n*** Subtable ",subtname
excllist = []
if subtname=='SOURCE':
excllist=['POSITION', 'TRANSITION', 'REST_FREQUENCY', 'SYSVEL']
if subtname=='SYSCAL':
excllist=['TANT_SPECTRUM', 'TANT_TSYS_SPECTRUM']
if subtname=='SPECTRAL_WINDOW':
excllist=['CHAN_FREQ', 'CHAN_WIDTH', 'EFFECTIVE_BW', 'RESOLUTION', 'ASSOC_SPW_ID', 'ASSOC_NATURE']
for colname in excllist:
if colname!='ASSOC_NATURE':
retValue['success'] = th.compVarColTables('reference.ms/SPECTRAL_WINDOW',
themsname+'/SPECTRAL_WINDOW', colname, 0.01) and retValue['success']
if subtname=='POLARIZATION':
excllist=['CORR_TYPE', 'CORR_PRODUCT']
for colname in excllist:
retValue['success'] = th.compVarColTables('reference.ms/POLARIZATION',
themsname+'/POLARIZATION', colname, 0.01) and retValue['success']
try:
retValue['success'] = th.compTables('reference.ms/'+subtname,
themsname+'/'+subtname, excllist,
0.01) and retValue['success']
except:
retValue['success'] = False
print "ERROR for table ", subtname
os.system("mv moved_"+myasdmname+" "+myasdmname)
self.assertTrue(retValue['success'],retValue['error_msgs'])
def test_mms7(self):
'''Asdm-import: Test good 12 m ASDM with mixed pol/channelisation input with default filler in lazy mode with reading the BDF flags. Output MMS'''
retValue = {'success': True, 'msgs': "", 'error_msgs': ''}
myasdmname = 'uid___A002_X71e4ae_X317_short'
themsname = myasdmname + ".ms"
self.res = importasdm(myasdmname,
vis=themsname,
lazy=True,
bdfflags=True,
createmms=True)
self.assertEqual(self.res, None)
print myname, ": Success! Now checking output ..."
mscomponents = set([
"ANTENNA/table.dat", "DATA_DESCRIPTION/table.dat",
"FEED/table.dat", "FIELD/table.dat", "FLAG_CMD/table.dat",
"HISTORY/table.dat", "OBSERVATION/table.dat", "POINTING/table.dat",
"POLARIZATION/table.dat", "PROCESSOR/table.dat",
"SOURCE/table.dat", "SPECTRAL_WINDOW/table.dat", "STATE/table.dat",
"SYSCAL/table.dat", "ANTENNA/table.f0",
"DATA_DESCRIPTION/table.f0", "FEED/table.f0", "FIELD/table.f0",
"FLAG_CMD/table.f0", "HISTORY/table.f0", "OBSERVATION/table.f0",
"POINTING/table.f0", "POLARIZATION/table.f0", "PROCESSOR/table.f0",
"SOURCE/table.f0", "SPECTRAL_WINDOW/table.f0", "STATE/table.f0",
"SYSCAL/table.f0"
])
for name in mscomponents:
if not os.access(themsname + "/" + name, os.F_OK):
print myname, ": Error ", themsname + "/" + name, "doesn't exist ..."
retValue['success'] = False
retValue['error_msgs'] = retValue[
'error_msgs'] + themsname + '/' + name + ' does not exist'
else:
print myname, ": ", name, "present."
print myname, ": MS exists. All tables present. Try opening as MS ..."
try:
mslocal.open(themsname)
except:
print myname, ": Error Cannot open MS table", themsname
retValue['success'] = False
retValue['error_msgs'] = retValue[
'error_msgs'] + 'Cannot open MS table ' + themsname
else:
mslocal.close()
print myname, ": OK. Checking tables in detail ..."
importasdm(asdm=myasdmname,
vis='reference.ms',
lazy=True,
overwrite=True,
bdfflags=False,
createmms=True)
if (os.path.exists('reference.ms')):
retValue['success'] = th.checkwithtaql(
"select from [select from reference.ms orderby TIME, DATA_DESC_ID, ANTENNA1, ANTENNA2 ] t1, [select from "
+ themsname +
" orderby TIME, DATA_DESC_ID, ANTENNA1, ANTENNA2 ] t2 where (not all(near(t1.DATA,t2.DATA, 1.e-06)))"
) == 0
if not retValue['success']:
print "ERROR: DATA does not agree with reference."
else:
print "DATA columns agree."
retValueTmp = th.checkwithtaql(
"select from [select from reference.ms orderby TIME, DATA_DESC_ID, ANTENNA1, ANTENNA2 ] t1, [select from "
+ themsname +
" orderby TIME, DATA_DESC_ID, ANTENNA1, ANTENNA2 ] t2 where (not all(t1.FLAG==t2.FLAG)) "
) != 0
if not retValueTmp:
print "ERROR: FLAG columns do agree with reference but they shouldn't."
else:
print "FLAG columns do not agree as expected."
retValue['success'] = retValue['success'] and retValueTmp
for subtname in [
"ANTENNA", "DATA_DESCRIPTION", "FEED", "FIELD",
"FLAG_CMD", "OBSERVATION", "POLARIZATION", "PROCESSOR",
"SOURCE", "SPECTRAL_WINDOW", "STATE", "SYSCAL"
]:
print "\n*** Subtable ", subtname
excllist = []
if subtname == 'SOURCE':
excllist = [
'POSITION', 'TRANSITION', 'REST_FREQUENCY',
'SYSVEL'
]
if subtname == 'SYSCAL':
excllist = ['TANT_SPECTRUM', 'TANT_TSYS_SPECTRUM']
if subtname == 'SPECTRAL_WINDOW':
excllist = [
'CHAN_FREQ', 'CHAN_WIDTH', 'EFFECTIVE_BW',
'RESOLUTION', 'ASSOC_SPW_ID', 'ASSOC_NATURE'
]
for colname in excllist:
if colname != 'ASSOC_NATURE':
retValue['success'] = th.compVarColTables(
'reference.ms/SPECTRAL_WINDOW',
themsname + '/SPECTRAL_WINDOW', colname,
0.01) and retValue['success']
if subtname == 'POLARIZATION':
excllist = ['CORR_TYPE', 'CORR_PRODUCT']
for colname in excllist:
retValue['success'] = th.compVarColTables(
'reference.ms/POLARIZATION',
themsname + '/POLARIZATION', colname,
0.01) and retValue['success']
try:
retValue['success'] = th.compTables(
'reference.ms/' + subtname, themsname + '/' +
subtname, excllist, 0.01) and retValue['success']
except:
retValue['success'] = False
print "ERROR for table ", subtname
self.assertTrue(retValue['success'], retValue['error_msgs'])
def test_mms4(self):
'''test_mms4: Create MMS, lazy=True, with separationaxis=scan and scans selection in ASDM'''
retValue = {'success': True, 'msgs': "", 'error_msgs': '' }
myasdmname = 'uid___A002_X72bc38_X000'
themsname = myasdmname + ".ms"
# only the first 3 scans to save time
importasdm(myasdmname, vis=themsname, lazy=True, scans='0:1~3', createmms=True, separationaxis='scan',
process_flags=False)
self.assertTrue(ParallelDataHelper.isParallelMS(themsname), 'Output is not a Multi-MS')
self.assertEqual(ph.axisType(themsname), 'scan', 'Separation axis of MMS should be scan')
print myname, ": Success! Now checking output ..."
mscomponents = set(["ANTENNA/table.dat",
"DATA_DESCRIPTION/table.dat",
"FEED/table.dat",
"FIELD/table.dat",
"FLAG_CMD/table.dat",
"HISTORY/table.dat",
"OBSERVATION/table.dat",
"POINTING/table.dat",
"POLARIZATION/table.dat",
"PROCESSOR/table.dat",
"SOURCE/table.dat",
"SPECTRAL_WINDOW/table.dat",
"STATE/table.dat",
"SYSCAL/table.dat",
"ANTENNA/table.f0",
"DATA_DESCRIPTION/table.f0",
"FEED/table.f0",
"FIELD/table.f0",
"FLAG_CMD/table.f0",
"HISTORY/table.f0",
"OBSERVATION/table.f0",
"POINTING/table.f0",
"POLARIZATION/table.f0",
"PROCESSOR/table.f0",
"SOURCE/table.f0",
"SPECTRAL_WINDOW/table.f0",
"STATE/table.f0",
"SYSCAL/table.f0"
])
for name in mscomponents:
if not os.access(themsname+"/"+name, os.F_OK):
print myname, ": Error ", themsname+"/"+name, "doesn't exist ..."
retValue['success']=False
retValue['error_msgs']=retValue['error_msgs']+themsname+'/'+name+' does not exist'
else:
print myname, ": ", name, "present."
print myname, ": MS exists. All tables present. Try opening as MS ..."
try:
mslocal.open(themsname)
except:
print myname, ": Error Cannot open MS table", themsname
retValue['success']=False
retValue['error_msgs']=retValue['error_msgs']+'Cannot open MS table '+themsname
else:
mslocal.close()
print myname, ": OK. Checking tables in detail ..."
importasdm(asdm=myasdmname, vis='reference.ms', lazy=False, overwrite=True, scans='0:1~3',
createmms=True, separationaxis='scan', process_flags=False)
if(os.path.exists('reference.ms')):
retValue['success'] = th.checkwithtaql("select from [select from reference.ms orderby TIME, DATA_DESC_ID, ANTENNA1, ANTENNA2 ] t1, [select from "
+themsname+" orderby TIME, DATA_DESC_ID, ANTENNA1, ANTENNA2 ] t2 where (not all(near(t1.DATA,t2.DATA, 1.e-06)))") == 0
if not retValue['success']:
print "ERROR: DATA does not agree with reference."
else:
print "DATA columns agree."
retValueTmp = th.checkwithtaql("select from [select from reference.ms orderby TIME, DATA_DESC_ID, ANTENNA1, ANTENNA2 ] t1, [select from "
+themsname+" orderby TIME, DATA_DESC_ID, ANTENNA1, ANTENNA2 ] t2 where (not all(t1.FLAG==t2.FLAG)) ") == 0
if not retValueTmp:
print "ERROR: FLAG does not agree with reference."
else:
print "FLAG columns agree."
retValue['success'] = retValue['success'] and retValueTmp
for subtname in ["ANTENNA",
"DATA_DESCRIPTION",
"FEED",
"FIELD",
"FLAG_CMD",
"OBSERVATION",
"POINTING",
"POLARIZATION",
"PROCESSOR",
"SOURCE",
"SPECTRAL_WINDOW",
"STATE",
"SYSCAL"]:
print "\n*** Subtable ",subtname
excllist = []
if subtname=='SOURCE':
excllist=['POSITION', 'TRANSITION', 'REST_FREQUENCY', 'SYSVEL']
if subtname=='SYSCAL':
excllist=['TANT_SPECTRUM', 'TANT_TSYS_SPECTRUM']
if subtname=='SPECTRAL_WINDOW':
excllist=['CHAN_FREQ', 'CHAN_WIDTH', 'EFFECTIVE_BW', 'RESOLUTION']
for colname in excllist:
retValue['success'] = th.compVarColTables('reference.ms/SPECTRAL_WINDOW',
themsname+'/SPECTRAL_WINDOW', colname, 0.01) and retValue['success']
try:
retValue['success'] = th.compTables('reference.ms/'+subtname,
themsname+'/'+subtname, excllist,
0.01) and retValue['success']
except:
retValue['success'] = False
print "ERROR for table ", subtname
print retValue
self.assertTrue(retValue['success'],retValue['error_msgs'])
def test_mms_ephem(self):
'''Asdm-import: Test good 12 m ASDM with Ephemeris table in lazy mode and MMS'''
retValue = {'success': True, 'msgs': "", 'error_msgs': ''}
myasdmname = 'uid___A002_X997a62_X8c-short'
themsname = myasdmname + ".ms"
self.res = importasdm(myasdmname,
vis=themsname,
lazy=True,
convert_ephem2geo=True,
process_pointing=False,
flagbackup=False,
createmms=True)
self.assertEqual(self.res, None)
print myname, ": Success! Now checking output ..."
mscomponents = set([
"FIELD/table.dat", "FIELD/EPHEM0_Mars_57034.9.tab",
"FIELD/EPHEM1_Titania_57034.9.tab"
])
for name in mscomponents:
if not os.access(themsname + "/" + name, os.F_OK):
print myname, ": Error ", themsname + "/" + name, "doesn't exist ..."
retValue['success'] = False
retValue['error_msgs'] = retValue[
'error_msgs'] + themsname + '/' + name + ' does not exist'
else:
print myname, ": ", name, "present."
print myname, ": MS exists. All relevant tables present. Try opening as MS ..."
try:
mslocal.open(themsname)
except:
print myname, ": Error Cannot open MS table", themsname
retValue['success'] = False
retValue['error_msgs'] = retValue[
'error_msgs'] + 'Cannot open MS table ' + themsname
else:
mslocal.close()
print myname, ": OK."
for name in [
"FIELD/EPHEM0_Mars_57034.9.tab",
"FIELD/EPHEM1_Titania_57034.9.tab"
]:
tblocal.open(themsname + "/" + name)
kw = tblocal.getkeywords()
tblocal.close()
geodist = kw['GeoDist'] # (km above reference ellipsoid)
geolat = kw['GeoLat'] # (deg)
geolong = kw['GeoLong'] # (deg)
print myname, ": Testing if ephemeris ", name, " was converted to GEO ..."
if not (geodist == geolat == geolong == 0.):
print myname, ": ERROR."
retValue['success'] = False
retValue['error_msgs'] = retValue[
'error_msgs'] + ' Ephemeris was not converted to GEO for ' + themsname + '\n'
else:
print myname, ": OK."
prsys = kw['posrefsys']
print myname, ": Testing if posrefsys was set correctly ..."
if not (prsys == "ICRF/ICRS"):
print myname, ": ERROR."
retValue['success'] = False
retValue['error_msgs'] = retValue[
'error_msgs'] + ' posrefsys keyword is not ICRF/ICRS ' + themsname + '\n'
else:
print myname, ": OK."
self.assertTrue(retValue['success'], retValue['error_msgs'])