def test_mms_scan_spw_partition(self):
'''mstransform: Create MMS and part by scan/spw'''
self.outputms = '3cscanspw02.mms'
mstransform(vis=self.vis, outputvis=self.outputms, datacolumn='data', spw='0,2',
createmms=True, disableparallel=True, separationaxis='auto')
# Verify the input versus the output
msmdt = msmdtool()
msmdt.open(self.outputms)
out_dds = msmdt.datadescids()
msmdt.done()
ref = [0,1,2]
for i in out_dds:
self.assertEqual(out_dds[i], ref[i])
# Verify that DATA_DESCRIPTION table is properly re-indexed.
spw_col = th.getVarCol(self.outputms+'/DATA_DESCRIPTION', 'SPECTRAL_WINDOW_ID')
self.assertEqual(spw_col.keys().__len__(), 3, 'Wrong number of rows in DD table')
self.assertEqual(spw_col['r1'][0], 0,'Error re-indexing SPECTRAL_WINDOW_ID of DATA_DESCRIPTION table')
self.assertEqual(spw_col['r2'][0], 0,'Error re-indexing SPECTRAL_WINDOW_ID of DATA_DESCRIPTION table')
self.assertEqual(spw_col['r3'][0], 1,'Error re-indexing SPECTRAL_WINDOW_ID of DATA_DESCRIPTION table')
# Check the FEED table
out_feed_spw = th.getVarCol(self.outputms+'/FEED', 'SPECTRAL_WINDOW_ID')
self.assertEqual(out_feed_spw['r1'],[0])
self.assertEqual(out_feed_spw['r26'],[0])
self.assertEqual(out_feed_spw['r27'],[1])
self.assertEqual(out_feed_spw['r28'],[1])
self.assertEqual(out_feed_spw['r51'],[1])
self.assertEqual(len(out_feed_spw.keys()), 52)
def test_mms_spw_selection3(self):
'''mstransform: Create MMS and select three spws with numsubms=2'''
self.outputms = '3cspw012.mms'
mstransform(vis=self.vis, outputvis=self.outputms, datacolumn='data', spw='0,1,2',
createmms=True, separationaxis='spw', numsubms=2)
# Verify the input versus the output
msmdt = msmdtool()
msmdt.open(self.outputms)
out_dds = msmdt.datadescids()
out_nrow = msmdt.nrows()
msmdt.done()
self.assertTrue(out_nrow,5200)
ref = [0,1,2,3]
for i in out_dds:
self.assertEqual(out_dds[i], ref[i])
# Verify that DATA_DESCRIPTION table is properly re-indexed.
spw_col = th.getVarCol(self.outputms+'/DATA_DESCRIPTION', 'SPECTRAL_WINDOW_ID')
self.assertEqual(spw_col.keys().__len__(), 4, 'Wrong number of rows in DD table')
self.assertEqual(spw_col['r1'][0], 0,'Error re-indexing SPECTRAL_WINDOW_ID of DATA_DESCRIPTION table')
self.assertEqual(spw_col['r2'][0], 0,'Error re-indexing SPECTRAL_WINDOW_ID of DATA_DESCRIPTION table')
self.assertEqual(spw_col['r3'][0], 1,'Error re-indexing SPECTRAL_WINDOW_ID of DATA_DESCRIPTION table')
self.assertEqual(spw_col['r4'][0], 2,'Error re-indexing SPECTRAL_WINDOW_ID of DATA_DESCRIPTION table')
in_feed_tb = th.getVarCol(self.vis+'/FEED', 'SPECTRAL_WINDOW_ID')
out_feed_tb = th.getVarCol(self.outputms+'/FEED', 'SPECTRAL_WINDOW_ID')
# Check the FEED table
th.compTables(self.vis+'/FEED', self.outputms+'/FEED', ['FOCUS_LENGTH'])
def test_mms_XXYY_selection(self):
'''mstransform: correlation='RR,LL' should select and re-index properly'''
self.outputms = '3cRRLL.mms'
# spw 0 should not be processed. The selection should happen before the MMS work
mstransform(vis=self.vis, outputvis=self.outputms, datacolumn='data', correlation='RR,LL',
createmms=True, separationaxis='auto')
msmdt = msmdtool()
msmdt.open(self.outputms)
out_dds = msmdt.datadescids()
msmdt.done()
ref = [0,1]
for i in out_dds:
self.assertEqual(out_dds[i], ref[i])
pol_col = th.getVarCol(self.outputms+'/POLARIZATION','NUM_CORR')
self.assertEqual(pol_col['r1'][0], 0,'Error in NUM_CORR of POLARIZATION table')
self.assertEqual(pol_col['r2'][0], 0,'Error in NUM_CORR of POLARIZATION table')
self.assertEqual(pol_col['r3'][0], 2,'Error in NUM_CORR of POLARIZATION table')
self.assertEqual(pol_col['r4'][0], 2,'Error in NUM_CORR of POLARIZATION table')
# Verify that POLARIZATION table is not re-sized.
corr_col = th.getVarCol(self.outputms+'/POLARIZATION', 'NUM_CORR')
self.assertEqual(corr_col.keys().__len__(), 4, 'Wrong number of rows in POLARIZATION table')
# Check the FEED table
# out_feed_spw = th.getVarCol(self.outputms+'/FEED', 'SPECTRAL_WINDOW_ID')
# self.assertEqual(len(out_feed_spw.keys()), 52)
# listobs, listpartition should not fail
listobs(self.outputms, listfile='3c_1.obs')
self.assertTrue(os.path.exists('3c_1.obs'), 'Probable error in sub-table re-indexing')
listpartition(self.outputms, listfile='3c_2.obs')
self.assertTrue(os.path.exists('3c_2.obs'), 'Probable error in sub-table re-indexing')
def test8(self):
"""hanningsmooth - Test 8: Flagging should be correct when hanning smoothing within mstransform (with regrid)"""
self.outputms = "cvelngc.ms"
# check correct flagging (just for one row as a sample)
flag_col = th.getVarCol(self.msfile, "FLAG")
self.assertTrue(flag_col["r1"][0][0] == [False])
self.assertTrue(flag_col["r1"][0][1] == [False])
self.assertTrue(flag_col["r1"][0][61] == [False])
self.assertTrue(flag_col["r1"][0][62] == [False])
# CAS-4114 cvel doesn't support MMS. Compare with mstransform instead.
# cvel(vis=self.msfile, outputvis=self.outputms, hanning=True, outframe='cmb')
mstransform(
vis=self.msfile, outputvis=self.outputms, datacolumn="data", hanning=True, regridms=True, outframe="cmb"
)
# check correct flagging (just for one row as a sample)
flag_col = th.getVarCol(self.outputms, "FLAG")
self.assertTrue(flag_col["r1"][0][0] == [True])
self.assertTrue(flag_col["r1"][0][1] == [False])
self.assertTrue(flag_col["r1"][0][2] == [False])
self.assertTrue(flag_col["r1"][0][60] == [False])
self.assertTrue(flag_col["r1"][0][61] == [True])
self.assertTrue(flag_col["r1"][0][62] == [True])
def test8(self):
'''hanningsmooth - Test 8: Flagging should be correct when hanning smoothing within mstransform (with regrid)'''
self.outputms = 'cvelngc.ms'
# check correct flagging (just for one row as a sample)
flag_col = th.getVarCol(self.msfile, 'FLAG')
self.assertTrue(flag_col['r1'][0][0] == [False])
self.assertTrue(flag_col['r1'][0][1] == [False])
self.assertTrue(flag_col['r1'][0][61] == [False])
self.assertTrue(flag_col['r1'][0][62] == [False])
# CAS-4114 cvel doesn't support MMS. Compare with mstransform instead.
# cvel(vis=self.msfile, outputvis=self.outputms, hanning=True, outframe='cmb')
mstransform(vis=self.msfile,
outputvis=self.outputms,
datacolumn='data',
hanning=True,
regridms=True,
outframe='cmb')
# check correct flagging (just for one row as a sample)
flag_col = th.getVarCol(self.outputms, 'FLAG')
self.assertTrue(flag_col['r1'][0][0] == [True])
self.assertTrue(flag_col['r1'][0][1] == [False])
self.assertTrue(flag_col['r1'][0][2] == [False])
self.assertTrue(flag_col['r1'][0][60] == [False])
self.assertTrue(flag_col['r1'][0][61] == [True])
self.assertTrue(flag_col['r1'][0][62] == [True])
def test_mms_spw_selection(self):
'''mstransform: Create MMS and select two spws with different polarization shapes'''
self.outputms = '3cspw12.mms'
mstransform(vis=self.vis, outputvis=self.outputms, datacolumn='data', spw='1,2',
createmms=True, separationaxis='spw')
# Verify the input versus the output
myms = mstool()
myms.open(self.vis)
myms.msselect({'spw':'1,2'})
inp_nrow = myms.nrow()
myms.close()
myms.open(self.outputms)
out_nrow = myms.nrow()
myms.close()
self.assertEqual(inp_nrow, out_nrow)
# Verify that DATA_DESCRIPTION table is properly re-indexed.
spw_col = th.getVarCol(self.outputms+'/DATA_DESCRIPTION', 'SPECTRAL_WINDOW_ID')
self.assertEqual(spw_col.keys().__len__(), 2, 'Wrong number of rows in DD table')
self.assertEqual(spw_col['r1'][0], 0,'Error re-indexing SPECTRAL_WINDOW_ID of DATA_DESCRIPTION table')
self.assertEqual(spw_col['r2'][0], 1,'Error re-indexing SPECTRAL_WINDOW_ID of DATA_DESCRIPTION table')
pol_col = th.getVarCol(self.outputms+'/DATA_DESCRIPTION', 'POLARIZATION_ID')
self.assertEqual(pol_col['r1'][0], 2,'Error in POLARIZATION_ID of DATA_DESCRIPTION table')
self.assertEqual(pol_col['r2'][0], 3,'Error in POLARIZATION_ID of DATA_DESCRIPTION table')
# Verify that POLARIZATION table is not re-sized.
corr_col = th.getVarCol(self.outputms+'/POLARIZATION', 'NUM_CORR')
self.assertEqual(corr_col.keys().__len__(), 4, 'Wrong number of rows in POLARIZATION table')
# Check the FEED table
out_feed_spw = th.getVarCol(self.outputms+'/FEED', 'SPECTRAL_WINDOW_ID')
self.assertEqual(len(out_feed_spw.keys()), 52)
def test_corrected_col(self):
'''hanningsmooth: Apply smoothing in CORRECTED column'''
self.outputms = 'hanncorr.ms'
# check correct flagging before (just for one row as a sample)
flag_col = th.getVarCol(self.msfile, 'FLAG')
self.assertTrue(flag_col['r1'][0][0] == [False])
self.assertTrue(flag_col['r1'][0][1] == [False])
self.assertTrue(flag_col['r1'][0][3838] == [False])
self.assertTrue(flag_col['r1'][0][3839] == [False])
# input column
data_col = th.getVarCol(self.msfile, 'CORRECTED_DATA')
hanningsmooth(vis=self.msfile,
outputvis=self.outputms,
datacolumn='corrected')
# output smoothed column
corr_col = th.getVarCol(self.outputms, 'DATA')
nrows = len(corr_col)
# check correct flagging after (just for one row as a sample)
flag_col = th.getVarCol(self.outputms, 'FLAG')
self.assertTrue(flag_col['r1'][0][0] == [True])
self.assertTrue(flag_col['r1'][0][1] == [False])
self.assertTrue(flag_col['r1'][0][3838] == [False])
self.assertTrue(flag_col['r1'][0][3839] == [True])
# Loop over every 2nd row,pol and get the data for each channel
max = 1e-04
for i in range(1, nrows, 2):
row = 'r%s' % i
# polarization is 0-1
for pol in range(0, 2):
# array's channels is 0-3840
for chan in range(1, 3839):
# channels must start from second and end before the last
data = data_col[row][pol][chan]
dataB = data_col[row][pol][chan - 1]
dataA = data_col[row][pol][chan + 1]
Smoothed = th.calculateHanning(dataB, data, dataA)
CorData = corr_col[row][pol][chan]
# Check the difference
self.assertTrue(
abs(CorData - Smoothed) < max,
'CorData=%s Smoothed=%s in row=%s pol=%s chan=%s' %
(CorData, Smoothed, row, pol, chan))
def test_corrected_col(self):
"""hanningsmooth: Apply smoothing in CORRECTED column"""
self.outputms = "hanncorr.ms"
# check correct flagging before (just for one row as a sample)
flag_col = th.getVarCol(self.msfile, "FLAG")
self.assertTrue(flag_col["r1"][0][0] == [False])
self.assertTrue(flag_col["r1"][0][1] == [False])
self.assertTrue(flag_col["r1"][0][3838] == [False])
self.assertTrue(flag_col["r1"][0][3839] == [False])
# input column
data_col = th.getVarCol(self.msfile, "CORRECTED_DATA")
hanningsmooth(vis=self.msfile, outputvis=self.outputms, datacolumn="corrected")
# output smoothed column
corr_col = th.getVarCol(self.outputms, "DATA")
nrows = len(corr_col)
# check correct flagging after (just for one row as a sample)
flag_col = th.getVarCol(self.outputms, "FLAG")
self.assertTrue(flag_col["r1"][0][0] == [True])
self.assertTrue(flag_col["r1"][0][1] == [False])
self.assertTrue(flag_col["r1"][0][3838] == [False])
self.assertTrue(flag_col["r1"][0][3839] == [True])
# Loop over every 2nd row,pol and get the data for each channel
max = 1e-04
for i in range(1, nrows, 2):
row = "r%s" % i
# polarization is 0-1
for pol in range(0, 2):
# array's channels is 0-3840
for chan in range(1, 3839):
# channels must start from second and end before the last
data = data_col[row][pol][chan]
dataB = data_col[row][pol][chan - 1]
dataA = data_col[row][pol][chan + 1]
Smoothed = th.calculateHanning(dataB, data, dataA)
CorData = corr_col[row][pol][chan]
# Check the difference
self.assertTrue(
abs(CorData - Smoothed) < max,
"CorData=%s Smoothed=%s in row=%s pol=%s chan=%s" % (CorData, Smoothed, row, pol, chan),
)
def test_freqavg7(self):
'''mstranform: Average using different bins for several spws, output MMS'''
# same as test_freqavg4
self.outputms = "favg7.ms"
mstransform(vis=self.vis, outputvis=self.outputms, spw='10,12,20', chanaverage=True,
chanbin=[128,4,10], createmms=True, separationaxis='scan', disableparallel=True)
self.assertTrue(os.path.exists(self.outputms))
# Output should be:
# spw=0 1 channel
# spw=1 32 channels
# spw=3 13 channels
ret = th.verifyMS(self.outputms, 3, 1, 0, ignoreflags=True)
self.assertTrue(ret[0],ret[1])
ret = th.verifyMS(self.outputms, 3, 32, 1, ignoreflags=True)
self.assertTrue(ret[0],ret[1])
ret = th.verifyMS(self.outputms, 3, 12, 2, ignoreflags=True)
self.assertTrue(ret[0],ret[1])
# Verify that some sub-tables are properly re-indexed.
spw_col = th.getVarCol(self.outputms+'/DATA_DESCRIPTION', 'SPECTRAL_WINDOW_ID')
self.assertEqual(spw_col.keys().__len__(), 3, 'Wrong number of rows in DD table')
self.assertEqual(spw_col['r1'][0], 0,'Error re-indexing DATA_DESCRIPTION table')
self.assertEqual(spw_col['r2'][0], 1,'Error re-indexing DATA_DESCRIPTION table')
self.assertEqual(spw_col['r3'][0], 2,'Error re-indexing DATA_DESCRIPTION table')
def test_freqavg10(self):
'''mstranform: Average using different bins, channel selection, both axes, output MMS'''
self.outputms = "favg10.ms"
mstransform(vis=self.vis, outputvis=self.outputms, spw='2,12,10:1~10', chanaverage=True,
chanbin=[32,128,5], createmms=True, separationaxis='auto', numsubms=6)
self.assertTrue(os.path.exists(self.outputms))
# Should create 6 subMSs
mslocal = mstool()
mslocal.open(thems=self.outputms)
sublist = mslocal.getreferencedtables()
mslocal.close()
self.assertEqual(sublist.__len__(), 6, 'Should have created 6 subMSs')
# Output should be:
# spw=0 4 channels
# spw=1 1 channel
# spw=2 2 channels
ret = th.verifyMS(self.outputms, 3, 4, 0, ignoreflags=True)
self.assertTrue(ret[0],ret[1])
ret = th.verifyMS(self.outputms, 3, 1, 1, ignoreflags=True)
self.assertTrue(ret[0],ret[1])
ret = th.verifyMS(self.outputms, 3, 2, 2, ignoreflags=True)
self.assertTrue(ret[0],ret[1])
# Verify that some sub-tables are properly re-indexed.
spw_col = th.getVarCol(self.outputms+'/DATA_DESCRIPTION', 'SPECTRAL_WINDOW_ID')
self.assertEqual(spw_col.keys().__len__(), 3, 'Wrong number of rows in DD table')
self.assertEqual(spw_col['r1'][0], 0,'Error re-indexing DATA_DESCRIPTION table')
self.assertEqual(spw_col['r2'][0], 1,'Error re-indexing DATA_DESCRIPTION table')
self.assertEqual(spw_col['r3'][0], 2,'Error re-indexing DATA_DESCRIPTION table')
def test_freqavg9(self):
'''mstranform: Average using different bins and a channel selection, output MMS'''
self.outputms = "favg9.ms"
mstransform(vis=self.vis, outputvis=self.outputms, spw='2,12,10:1~10', chanaverage=True,
chanbin=[32,128,5], createmms=True, separationaxis='spw')
self.assertTrue(os.path.exists(self.outputms))
# Output should be:
# spw=0 4 channels
# spw=1 1 channel
# spw=2 2 channels
ret = th.verifyMS(self.outputms, 3, 4, 0, ignoreflags=True)
self.assertTrue(ret[0],ret[1])
ret = th.verifyMS(self.outputms, 3, 1, 1, ignoreflags=True)
self.assertTrue(ret[0],ret[1])
ret = th.verifyMS(self.outputms, 3, 2, 2, ignoreflags=True)
self.assertTrue(ret[0],ret[1])
# Verify that some sub-tables are properly re-indexed.
spw_col = th.getVarCol(self.outputms+'/DATA_DESCRIPTION', 'SPECTRAL_WINDOW_ID')
self.assertEqual(spw_col.keys().__len__(), 3, 'Wrong number of rows in DD table')
self.assertEqual(spw_col['r1'][0], 0,'Error re-indexing DATA_DESCRIPTION table')
self.assertEqual(spw_col['r2'][0], 1,'Error re-indexing DATA_DESCRIPTION table')
self.assertEqual(spw_col['r3'][0], 2,'Error re-indexing DATA_DESCRIPTION table')
def test_regrid1_3(self):
'''mstransform: Default regridms with spw selection using input MMS'''
# same as test_regrid1_1
mmsfile = 'testmms13.mms'
# Create input MMS
mstransform(vis=self.vis, outputvis=mmsfile, createmms=True, disableparallel=True,
separationaxis='scan')
self.outputms = "reg13.ms"
mstransform(vis=mmsfile, outputvis=self.outputms, regridms=True, spw='1,3,5,7',
datacolumn='DATA')
self.assertTrue(os.path.exists(self.outputms))
# The regriding should be the same as the input
for i in range(4):
ret = th.verifyMS(self.outputms, 4, 64, i)
self.assertTrue(ret[0],ret[1])
listobs(self.outputms)
# Verify that some sub-tables are properly re-indexed.
spw_col = th.getVarCol(self.outputms+'/DATA_DESCRIPTION', 'SPECTRAL_WINDOW_ID')
self.assertEqual(spw_col.keys().__len__(), 4, 'Wrong number of rows in DD table')
self.assertEqual(spw_col['r1'][0], 0,'Error re-indexing DATA_DESCRIPTION table')
self.assertEqual(spw_col['r2'][0], 1,'Error re-indexing DATA_DESCRIPTION table')
self.assertEqual(spw_col['r3'][0], 2,'Error re-indexing DATA_DESCRIPTION table')
self.assertEqual(spw_col['r4'][0], 3,'Error re-indexing DATA_DESCRIPTION table')
def test3(self):
'''hanningsmooth - Test 3: Check theoretical and calculated values on non-existing CORRECTED column'''
self.outputms = 'hann3.ms'
# check correct flagging (just for one row as a sample)
flag_col = th.getVarCol(self.msfile, 'FLAG')
self.assertTrue(flag_col['r1'][0][0] == [False])
self.assertTrue(flag_col['r1'][0][1] == [False])
self.assertTrue(flag_col['r1'][0][61] == [False])
self.assertTrue(flag_col['r1'][0][62] == [False])
# It should fall-back and use the input DATA column
hanningsmooth(vis=self.msfile,
outputvis=self.outputms,
datacolumn='corrected')
# check correct flagging (just for one row as a sample)
flag_col = th.getVarCol(self.outputms, 'FLAG')
self.assertTrue(flag_col['r1'][0][0] == [True])
self.assertTrue(flag_col['r1'][0][1] == [False])
self.assertTrue(flag_col['r1'][0][61] == [False])
self.assertTrue(flag_col['r1'][0][62] == [True])
data_col = th.getVarCol(self.msfile, 'DATA')
corr_col = th.getVarCol(self.outputms, 'DATA')
nrows = len(corr_col)
# Loop over every 2nd row,pol and get the data for each channel
max = 1e-05
for i in range(1, nrows, 2):
row = 'r%s' % i
# polarization is 0-1
for pol in range(0, 2):
# array's channels is 0-63
for chan in range(1, 62):
# channels must start from second and end before the last
data = data_col[row][pol][chan]
dataB = data_col[row][pol][chan - 1]
dataA = data_col[row][pol][chan + 1]
Smoothed = th.calculateHanning(dataB, data, dataA)
CorData = corr_col[row][pol][chan]
# Check the difference
self.assertTrue(abs(CorData - Smoothed) < max)
def test6(self):
"""hanningsmooth - Test 6: Flagging should be correct with datacolumn==ALL"""
self.outputms = "hann6.ms"
# check correct flagging (just for one row as a sample)
flag_col = th.getVarCol(self.msfile, "FLAG")
self.assertTrue(flag_col["r1"][0][0] == [False])
self.assertTrue(flag_col["r1"][0][1] == [False])
self.assertTrue(flag_col["r1"][0][61] == [False])
self.assertTrue(flag_col["r1"][0][62] == [False])
hanningsmooth(vis=self.msfile, outputvis=self.outputms, datacolumn="all")
# check correct flagging (just for one row as a sample)
flag_col = th.getVarCol(self.outputms, "FLAG")
self.assertTrue(flag_col["r1"][0][0] == [True])
self.assertTrue(flag_col["r1"][0][1] == [False])
self.assertTrue(flag_col["r1"][0][61] == [False])
self.assertTrue(flag_col["r1"][0][62] == [True])
def test7(self):
'''Test 7: Flagging should be correct when hanning smoothing within cvel (no transform)'''
clearcal(vis=self.msfile)
# check correct flagging (just for one row as a sample)
flag_col = th.getVarCol(self.msfile, 'FLAG')
self.assertTrue(flag_col['r1'][0][0] == [False])
self.assertTrue(flag_col['r1'][0][1] == [False])
self.assertTrue(flag_col['r1'][0][61] == [False])
self.assertTrue(flag_col['r1'][0][62] == [False])
self.res = cvel(vis=self.msfile, outputvis='cvelngc.ms', hanning=True)
# check correct flagging (just for one row as a sample)
flag_col = th.getVarCol('cvelngc.ms', 'FLAG')
self.assertTrue(flag_col['r1'][0][0] == [True])
self.assertTrue(flag_col['r1'][0][1] == [False])
self.assertTrue(flag_col['r1'][0][61] == [False])
self.assertTrue(flag_col['r1'][0][62] == [True])
def test6(self):
'''Test 6: Flagging should be correct with datacolumn==ALL'''
clearcal(vis=self.msfile)
# check correct flagging (just for one row as a sample)
flag_col = th.getVarCol(self.msfile, 'FLAG')
self.assertTrue(flag_col['r1'][0][0] == [False])
self.assertTrue(flag_col['r1'][0][1] == [False])
self.assertTrue(flag_col['r1'][0][61] == [False])
self.assertTrue(flag_col['r1'][0][62] == [False])
self.res = oldhanningsmooth(vis=self.msfile, datacolumn='all')
# check correct flagging (just for one row as a sample)
flag_col = th.getVarCol(self.msfile, 'FLAG')
self.assertTrue(flag_col['r1'][0][0] == [True])
self.assertTrue(flag_col['r1'][0][1] == [False])
self.assertTrue(flag_col['r1'][0][61] == [False])
self.assertTrue(flag_col['r1'][0][62] == [True])
def test3(self):
'''hanningsmooth2 - Test 3: Check theoretical and calculated values on non-existing CORRECTED column'''
self.outputms = 'hann3.ms'
# check correct flagging (just for one row as a sample)
flag_col = th.getVarCol(self.msfile, 'FLAG')
self.assertTrue(flag_col['r1'][0][0] == [False])
self.assertTrue(flag_col['r1'][0][1] == [False])
self.assertTrue(flag_col['r1'][0][61] == [False])
self.assertTrue(flag_col['r1'][0][62] == [False])
# It should fall-back and use the input DATA column
hanningsmooth2(vis=self.msfile, outputvis=self.outputms, datacolumn='corrected')
# check correct flagging (just for one row as a sample)
flag_col = th.getVarCol(self.outputms, 'FLAG')
self.assertTrue(flag_col['r1'][0][0] == [True])
self.assertTrue(flag_col['r1'][0][1] == [False])
self.assertTrue(flag_col['r1'][0][61] == [False])
self.assertTrue(flag_col['r1'][0][62] == [True])
data_col = th.getVarCol(self.msfile, 'DATA')
corr_col = th.getVarCol(self.outputms, 'DATA')
nrows = len(corr_col)
# Loop over every 2nd row,pol and get the data for each channel
max = 1e-05
for i in range(1,nrows,2) :
row = 'r%s'%i
# polarization is 0-1
for pol in range(0,2) :
# array's channels is 0-63
for chan in range(1,62) :
# channels must start from second and end before the last
data = data_col[row][pol][chan]
dataB = data_col[row][pol][chan-1]
dataA = data_col[row][pol][chan+1]
Smoothed = th.calculateHanning(dataB,data,dataA)
CorData = corr_col[row][pol][chan]
# Check the difference
self.assertTrue(abs(CorData-Smoothed) < max )
def test6(self):
'''Test 6: Flagging should be correct with datacolumn==ALL'''
clearcal(vis=self.msfile)
# check correct flagging (just for one row as a sample)
flag_col = th.getVarCol(self.msfile, 'FLAG')
self.assertTrue(flag_col['r1'][0][0] == [False])
self.assertTrue(flag_col['r1'][0][1] == [False])
self.assertTrue(flag_col['r1'][0][61] == [False])
self.assertTrue(flag_col['r1'][0][62] == [False])
self.res = hanningsmooth(vis=self.msfile,datacolumn='all')
# check correct flagging (just for one row as a sample)
flag_col = th.getVarCol(self.msfile, 'FLAG')
self.assertTrue(flag_col['r1'][0][0] == [True])
self.assertTrue(flag_col['r1'][0][1] == [False])
self.assertTrue(flag_col['r1'][0][61] == [False])
self.assertTrue(flag_col['r1'][0][62] == [True])
def test6(self):
'''hanningsmooth2 - Test 6: Flagging should be correct with datacolumn==ALL'''
self.outputms = 'hann6.ms'
# check correct flagging (just for one row as a sample)
flag_col = th.getVarCol(self.msfile, 'FLAG')
self.assertTrue(flag_col['r1'][0][0] == [False])
self.assertTrue(flag_col['r1'][0][1] == [False])
self.assertTrue(flag_col['r1'][0][61] == [False])
self.assertTrue(flag_col['r1'][0][62] == [False])
hanningsmooth2(vis=self.msfile,outputvis=self.outputms, datacolumn='all')
# check correct flagging (just for one row as a sample)
flag_col = th.getVarCol(self.outputms, 'FLAG')
self.assertTrue(flag_col['r1'][0][0] == [True])
self.assertTrue(flag_col['r1'][0][1] == [False])
self.assertTrue(flag_col['r1'][0][61] == [False])
self.assertTrue(flag_col['r1'][0][62] == [True])
def test3(self):
'''Test 3: Theoretical and calculated values should be the same with datacolumn==CORRECTED'''
# check correct flagging (just for one row as a sample)
flag_col = th.getVarCol(self.msfile, 'FLAG')
self.assertTrue(flag_col['r1'][0][0] == [False])
self.assertTrue(flag_col['r1'][0][1] == [False])
self.assertTrue(flag_col['r1'][0][61] == [False])
self.assertTrue(flag_col['r1'][0][62] == [False])
self.res = oldhanningsmooth(vis=self.msfile, datacolumn='corrected')
# check correct flagging (just for one row as a sample)
flag_col = th.getVarCol(self.msfile, 'FLAG')
self.assertTrue(flag_col['r1'][0][0] == [True])
self.assertTrue(flag_col['r1'][0][1] == [False])
self.assertTrue(flag_col['r1'][0][61] == [False])
self.assertTrue(flag_col['r1'][0][62] == [True])
cd = th.getColDesc(self.msfile, 'CORRECTED_DATA')
self.assertTrue(len(cd))
data_col = th.getVarCol(self.msfile, 'DATA')
corr_col = th.getVarCol(self.msfile, 'CORRECTED_DATA')
nrows = len(corr_col)
# Loop over every 2nd row,pol and get the data for each channel
max = 1e-05
for i in range(1, nrows, 2):
row = 'r%s' % i
# polarization is 0-1
for pol in range(0, 2):
# array's channels is 0-63
for chan in range(1, 62):
# channels must start from second and end before the last
data = data_col[row][pol][chan]
dataB = data_col[row][pol][chan - 1]
dataA = data_col[row][pol][chan + 1]
Smoothed = th.calculateHanning(dataB, data, dataA)
CorData = corr_col[row][pol][chan]
# Check the difference
self.assertTrue(abs(CorData - Smoothed) < max)
def test7(self):
'''Test 7: Flagging should be correct when hanning smoothing within cvel (no transform)'''
clearcal(vis=self.msfile)
# check correct flagging (just for one row as a sample)
flag_col = th.getVarCol(self.msfile, 'FLAG')
self.assertTrue(flag_col['r1'][0][0] == [False])
self.assertTrue(flag_col['r1'][0][1] == [False])
self.assertTrue(flag_col['r1'][0][61] == [False])
self.assertTrue(flag_col['r1'][0][62] == [False])
self.res = cvel(vis=self.msfile, outputvis='cvelngc.ms', hanning=True)
# check correct flagging (just for one row as a sample)
flag_col = th.getVarCol('cvelngc.ms', 'FLAG')
self.assertTrue(flag_col['r1'][0][0] == [True])
self.assertTrue(flag_col['r1'][0][1] == [False])
self.assertTrue(flag_col['r1'][0][61] == [False])
self.assertTrue(flag_col['r1'][0][62] == [True])
def test3(self):
'''Test 3: Theoretical and calculated values should be the same with datacolumn==CORRECTED'''
# check correct flagging (just for one row as a sample)
flag_col = th.getVarCol(self.msfile, 'FLAG')
self.assertTrue(flag_col['r1'][0][0] == [False])
self.assertTrue(flag_col['r1'][0][1] == [False])
self.assertTrue(flag_col['r1'][0][61] == [False])
self.assertTrue(flag_col['r1'][0][62] == [False])
self.res = hanningsmooth(vis=self.msfile, datacolumn='corrected')
# check correct flagging (just for one row as a sample)
flag_col = th.getVarCol(self.msfile, 'FLAG')
self.assertTrue(flag_col['r1'][0][0] == [True])
self.assertTrue(flag_col['r1'][0][1] == [False])
self.assertTrue(flag_col['r1'][0][61] == [False])
self.assertTrue(flag_col['r1'][0][62] == [True])
cd = th.getColDesc(self.msfile, 'CORRECTED_DATA')
self.assertTrue(len(cd))
data_col = th.getVarCol(self.msfile, 'DATA')
corr_col = th.getVarCol(self.msfile, 'CORRECTED_DATA')
nrows = len(corr_col)
# Loop over every 2nd row,pol and get the data for each channel
max = 1e-05
for i in range(1,nrows,2) :
row = 'r%s'%i
# polarization is 0-1
for pol in range(0,2) :
# array's channels is 0-63
for chan in range(1,62) :
# channels must start from second and end before the last
data = data_col[row][pol][chan]
dataB = data_col[row][pol][chan-1]
dataA = data_col[row][pol][chan+1]
Smoothed = th.calculateHanning(dataB,data,dataA)
CorData = corr_col[row][pol][chan]
# Check the difference
self.assertTrue(abs(CorData-Smoothed) < max )
def test7(self):
"""hanningsmooth - Test 7: Flagging should be correct when hanning smoothing within cvel (no transform)"""
self.outputms = "cvelngc.ms"
clearcal(vis=self.msfile)
# check correct flagging (just for one row as a sample)
flag_col = th.getVarCol(self.msfile, "FLAG")
self.assertTrue(flag_col["r1"][0][0] == [False])
self.assertTrue(flag_col["r1"][0][1] == [False])
self.assertTrue(flag_col["r1"][0][61] == [False])
self.assertTrue(flag_col["r1"][0][62] == [False])
cvel(vis=self.msfile, outputvis=self.outputms, hanning=True)
# check correct flagging (just for one row as a sample)
flag_col = th.getVarCol(self.outputms, "FLAG")
self.assertTrue(flag_col["r1"][0][0] == [True])
self.assertTrue(flag_col["r1"][0][1] == [False])
self.assertTrue(flag_col["r1"][0][61] == [False])
self.assertTrue(flag_col["r1"][0][62] == [True])
def test6(self):
'''hanningsmooth - Test 6: Flagging should be correct with datacolumn==ALL'''
self.outputms = 'hann6.ms'
# check correct flagging (just for one row as a sample)
flag_col = th.getVarCol(self.msfile, 'FLAG')
self.assertTrue(flag_col['r1'][0][0] == [False])
self.assertTrue(flag_col['r1'][0][1] == [False])
self.assertTrue(flag_col['r1'][0][61] == [False])
self.assertTrue(flag_col['r1'][0][62] == [False])
hanningsmooth(vis=self.msfile,
outputvis=self.outputms,
datacolumn='all')
# check correct flagging (just for one row as a sample)
flag_col = th.getVarCol(self.outputms, 'FLAG')
self.assertTrue(flag_col['r1'][0][0] == [True])
self.assertTrue(flag_col['r1'][0][1] == [False])
self.assertTrue(flag_col['r1'][0][61] == [False])
self.assertTrue(flag_col['r1'][0][62] == [True])
def test8(self):
'''hanningsmooth2 - Test 8: Flagging should be correct when hanning smoothing within mstransform (with regrid)'''
self.outputms = 'cvelngc.ms'
# check correct flagging (just for one row as a sample)
flag_col = th.getVarCol(self.msfile, 'FLAG')
self.assertTrue(flag_col['r1'][0][0] == [False])
self.assertTrue(flag_col['r1'][0][1] == [False])
self.assertTrue(flag_col['r1'][0][61] == [False])
self.assertTrue(flag_col['r1'][0][62] == [False])
# CAS-4114 cvel doesn't support MMS. Compare with mstransform instead.
# cvel(vis=self.msfile, outputvis=self.outputms, hanning=True, outframe='cmb')
mstransform(vis=self.msfile, outputvis=self.outputms, datacolumn='data',
hanning=True, regridms=True, outframe='cmb')
# check correct flagging (just for one row as a sample)
flag_col = th.getVarCol(self.outputms, 'FLAG')
self.assertTrue(flag_col['r1'][0][0] == [True])
self.assertTrue(flag_col['r1'][0][1] == [False])
self.assertTrue(flag_col['r1'][0][2] == [False])
self.assertTrue(flag_col['r1'][0][60] == [False])
self.assertTrue(flag_col['r1'][0][61] == [True])
self.assertTrue(flag_col['r1'][0][62] == [True])
def test_parallel4(self):
'''mstransform: verify spw sub-table consolidation in sequential'''
self.outputms = "parallel4.mms"
mstransform(vis=self.vis, outputvis=self.outputms, spw='3,5:10~20,7,9,15',createmms=True,
separationaxis='spw', numsubms=5)
self.assertTrue(os.path.exists(self.outputms))
# spw=5 should be spw=1 after consolidation, with 10 channels
ret = th.verifyMS(self.outputms, 7, 10, 1, ignoreflags=True)
# Check the FEED table
out_feed_spw = th.getVarCol(self.outputms+'/FEED', 'SPECTRAL_WINDOW_ID')
self.assertEqual(len(out_feed_spw.keys()), 20)
self.assertEqual(out_feed_spw['r1'], 0)
self.assertEqual(out_feed_spw['r5'], 1)
self.assertEqual(out_feed_spw['r9'], 2)
self.assertEqual(out_feed_spw['r13'], 3)
self.assertEqual(out_feed_spw['r17'], 4)
def test_parallel2(self):
'''mstransform: create MMS with spw/scan separation and channel selections in parallel'''
self.outputms = "parallel2.mms"
mstransform(vis=self.vis, outputvis=self.outputms, spw='0:0~10,1:60~63',createmms=True,
separationaxis='auto')
self.assertTrue(os.path.exists(self.outputms))
# It should create 4 subMS, with spw=0~1
# spw=0 has 11 channels, spw=1 has 4 channels
ret = th.verifyMS(self.outputms, 2, 11, 0, ignoreflags=True)
self.assertTrue(ret[0],ret[1])
ret = th.verifyMS(self.outputms, 2, 4, 1, ignoreflags=True)
self.assertTrue(ret[0],ret[1])
# Check the FEED table
out_feed_spw = th.getVarCol(self.outputms+'/FEED', 'SPECTRAL_WINDOW_ID')
self.assertEqual(len(out_feed_spw.keys()), 8)
self.assertEqual(out_feed_spw['r1'], 0)
self.assertEqual(out_feed_spw['r5'], 1)
def test_channels_mms2(self):
'''mstransform: create MMS with spw/scan separation and channel selections'''
self.outputms = "testmms2.mms"
mstransform(vis=self.vis, outputvis=self.outputms, spw='0:0~10,1:60~63',createmms=True,
separationaxis='auto', disableparallel=True)
self.assertTrue(os.path.exists(self.outputms))
# It should create 4 subMS, with spw=0~1
# spw=0 has 11 channels, spw=1 has 4 channels
ret = th.verifyMS(self.outputms, 2, 11, 0, ignoreflags=True)
self.assertTrue(ret[0],ret[1])
ret = th.verifyMS(self.outputms, 2, 4, 1, ignoreflags=True)
self.assertTrue(ret[0],ret[1])
# Verify that some sub-tables are properly re-indexed.
spw_col = th.getVarCol(self.outputms+'/DATA_DESCRIPTION', 'SPECTRAL_WINDOW_ID')
self.assertEqual(spw_col.keys().__len__(), 2, 'Wrong number of rows in DD table')
self.assertEqual(spw_col['r1'][0], 0,'Error re-indexing DATA_DESCRIPTION table')
self.assertEqual(spw_col['r2'][0], 1,'Error re-indexing DATA_DESCRIPTION table')
# The separation axis should be written to the output MMS
sepaxis = ph.axisType(self.outputms)
self.assertEqual(sepaxis, 'scan,spw', 'AxisType is not correctly written to output MMS')
def test4(self):
'''hanningsmooth2 - Test 4: Theoretical and calculated values should be the same for MMS-case'''
# Split the input to decrease the running time
split2(self.msfile, outputvis='splithan.ms',scan='1,2',datacolumn='data')
self.msfile = 'splithan.ms'
# create a test MMS. It creates self.testmms
self.createMMS(self.msfile)
self.outputms = 'hann4.mms'
# check correct flagging (just for one row as a sample)
mslocal = mstool()
mslocal.open(self.msfile)
mslocal.sort('sorted.ms',['OBSERVATION_ID','ARRAY_ID','SCAN_NUMBER','FIELD_ID','DATA_DESC_ID','ANTENNA1','ANTENNA2','TIME'])
mslocal.close()
self.msfile = 'sorted.ms'
flag_col = th.getVarCol(self.msfile, 'FLAG')
self.assertTrue(flag_col['r1'][0][0] == [False])
self.assertTrue(flag_col['r1'][0][1] == [False])
self.assertTrue(flag_col['r1'][0][61] == [False])
self.assertTrue(flag_col['r1'][0][62] == [False])
data_col = th.getVarCol(self.msfile, 'DATA')
hanningsmooth2(vis=self.testmms, outputvis=self.outputms, datacolumn='data', keepmms=True)
self.assertTrue(ParallelDataHelper.isParallelMS(self.outputms), 'Output should be an MMS')
# Sort the MMS
mslocal.open(self.outputms)
mslocal.sort('sorted.mms',['OBSERVATION_ID','ARRAY_ID','SCAN_NUMBER','FIELD_ID','DATA_DESC_ID','ANTENNA1','ANTENNA2','TIME'])
mslocal.close()
self.outputms = 'sorted.mms'
corr_col = th.getVarCol(self.outputms, 'DATA')
nrows = len(corr_col)
# check correct flagging (just for one row as a sample)
flag_col = th.getVarCol(self.outputms, 'FLAG')
self.assertTrue(flag_col['r1'][0][0] == [True])
self.assertTrue(flag_col['r1'][0][1] == [False])
self.assertTrue(flag_col['r1'][0][61] == [False])
self.assertTrue(flag_col['r1'][0][62] == [True])
# Loop over every 2nd row,pol and get the data for each channel
max = 1e-05
for i in range(1,nrows,2) :
row = 'r%s'%i
# polarization is 0-1
for pol in range(0,2) :
# array's channels is 0-63
for chan in range(1,62) :
# channels must start from second and end before the last
data = data_col[row][pol][chan]
dataB = data_col[row][pol][chan-1]
dataA = data_col[row][pol][chan+1]
Smoothed = th.calculateHanning(dataB,data,dataA)
CorData = corr_col[row][pol][chan]
# Check the difference
self.assertTrue(abs(CorData-Smoothed) < max )
def test4(self):
"""hanningsmooth - Test 4: Theoretical and calculated values should be the same for MMS-case"""
# Split the input to decrease the running time
split(self.msfile, outputvis="splithan.ms", scan="1,2", datacolumn="data")
self.msfile = "splithan.ms"
# create a test MMS. It creates self.testmms
self.createMMS(self.msfile)
self.outputms = "hann4.mms"
# check correct flagging (just for one row as a sample)
mslocal = mstool()
mslocal.open(self.msfile)
mslocal.sort(
"sorted.ms",
["OBSERVATION_ID", "ARRAY_ID", "SCAN_NUMBER", "FIELD_ID", "DATA_DESC_ID", "ANTENNA1", "ANTENNA2", "TIME"],
)
mslocal.close()
self.msfile = "sorted.ms"
flag_col = th.getVarCol(self.msfile, "FLAG")
self.assertTrue(flag_col["r1"][0][0] == [False])
self.assertTrue(flag_col["r1"][0][1] == [False])
self.assertTrue(flag_col["r1"][0][61] == [False])
self.assertTrue(flag_col["r1"][0][62] == [False])
data_col = th.getVarCol(self.msfile, "DATA")
hanningsmooth(vis=self.testmms, outputvis=self.outputms, datacolumn="data", keepmms=True)
self.assertTrue(ParallelDataHelper.isParallelMS(self.outputms), "Output should be an MMS")
# Sort the MMS
mslocal.open(self.outputms)
mslocal.sort(
"sorted.mms",
["OBSERVATION_ID", "ARRAY_ID", "SCAN_NUMBER", "FIELD_ID", "DATA_DESC_ID", "ANTENNA1", "ANTENNA2", "TIME"],
)
mslocal.close()
self.outputms = "sorted.mms"
corr_col = th.getVarCol(self.outputms, "DATA")
nrows = len(corr_col)
# check correct flagging (just for one row as a sample)
flag_col = th.getVarCol(self.outputms, "FLAG")
self.assertTrue(flag_col["r1"][0][0] == [True])
self.assertTrue(flag_col["r1"][0][1] == [False])
self.assertTrue(flag_col["r1"][0][61] == [False])
self.assertTrue(flag_col["r1"][0][62] == [True])
# Loop over every 2nd row,pol and get the data for each channel
max = 1e-05
for i in range(1, nrows, 2):
row = "r%s" % i
# polarization is 0-1
for pol in range(0, 2):
# array's channels is 0-63
for chan in range(1, 62):
# channels must start from second and end before the last
data = data_col[row][pol][chan]
dataB = data_col[row][pol][chan - 1]
dataA = data_col[row][pol][chan + 1]
Smoothed = th.calculateHanning(dataB, data, dataA)
CorData = corr_col[row][pol][chan]
# Check the difference
self.assertTrue(abs(CorData - Smoothed) < max)
def test4(self):
'''hanningsmooth - Test 4: Theoretical and calculated values should be the same for MMS-case'''
# Split the input to decrease the running time
split(self.msfile,
outputvis='splithan.ms',
scan='1,2',
datacolumn='data')
self.msfile = 'splithan.ms'
# create a test MMS. It creates self.testmms
self.createMMS(self.msfile)
self.outputms = 'hann4.mms'
# check correct flagging (just for one row as a sample)
mslocal = mstool()
mslocal.open(self.msfile)
mslocal.sort('sorted.ms', [
'OBSERVATION_ID', 'ARRAY_ID', 'SCAN_NUMBER', 'FIELD_ID',
'DATA_DESC_ID', 'ANTENNA1', 'ANTENNA2', 'TIME'
])
mslocal.close()
self.msfile = 'sorted.ms'
flag_col = th.getVarCol(self.msfile, 'FLAG')
self.assertTrue(flag_col['r1'][0][0] == [False])
self.assertTrue(flag_col['r1'][0][1] == [False])
self.assertTrue(flag_col['r1'][0][61] == [False])
self.assertTrue(flag_col['r1'][0][62] == [False])
data_col = th.getVarCol(self.msfile, 'DATA')
hanningsmooth(vis=self.testmms,
outputvis=self.outputms,
datacolumn='data',
keepmms=True)
self.assertTrue(ParallelDataHelper.isParallelMS(self.outputms),
'Output should be an MMS')
# Sort the MMS
mslocal.open(self.outputms)
mslocal.sort('sorted.mms', [
'OBSERVATION_ID', 'ARRAY_ID', 'SCAN_NUMBER', 'FIELD_ID',
'DATA_DESC_ID', 'ANTENNA1', 'ANTENNA2', 'TIME'
])
mslocal.close()
self.outputms = 'sorted.mms'
corr_col = th.getVarCol(self.outputms, 'DATA')
nrows = len(corr_col)
# check correct flagging (just for one row as a sample)
flag_col = th.getVarCol(self.outputms, 'FLAG')
self.assertTrue(flag_col['r1'][0][0] == [True])
self.assertTrue(flag_col['r1'][0][1] == [False])
self.assertTrue(flag_col['r1'][0][61] == [False])
self.assertTrue(flag_col['r1'][0][62] == [True])
# Loop over every 2nd row,pol and get the data for each channel
max = 1e-05
for i in range(1, nrows, 2):
row = 'r%s' % i
# polarization is 0-1
for pol in range(0, 2):
# array's channels is 0-63
for chan in range(1, 62):
# channels must start from second and end before the last
data = data_col[row][pol][chan]
dataB = data_col[row][pol][chan - 1]
dataA = data_col[row][pol][chan + 1]
Smoothed = th.calculateHanning(dataB, data, dataA)
CorData = corr_col[row][pol][chan]
# Check the difference
self.assertTrue(abs(CorData - Smoothed) < max)
