def test6(self):
'''cvel2 6: I/O vis set, more complex input vis, one field selected, one spw selected, passall = True'''
if testmms:
return
self.setUp_vis_a()
myvis = vis_a
os.system('ln -sf ' + myvis + ' myinput.ms')
rval = cvel2(
vis = 'myinput.ms',
outputvis = outfile,
field = '1',
spw = '0',
nchan = 32,
start = 10,
passall = True
)
self.assertNotEqual(rval,False)
# Simulate the passall=True. This MS has fields 0~6
desel = outfile+'.deselected'
split(vis='myinput.ms',outputvis=desel,field='0,2,3,4,5,6',spw='0',datacolumn='all')
mslocal = mstool()
mslocal.open(outfile, nomodify=False)
mslocal.concatenate(msfile=desel)
mslocal.close()
ret = (verify_ms(outfile, 2, 32, 0))
self.assertTrue(ret[0],ret[1])
def test6(self):
'''cvel2 6: I/O vis set, more complex input vis, one field selected, one spw selected, passall = True'''
if testmms:
return
self.setUp_vis_a()
myvis = vis_a
os.system('ln -sf ' + myvis + ' myinput.ms')
rval = cvel2(vis='myinput.ms',
outputvis=outfile,
field='1',
spw='0',
nchan=32,
start=10,
passall=True)
self.assertNotEqual(rval, False)
# Simulate the passall=True. This MS has fields 0~6
desel = outfile + '.deselected'
split(vis='myinput.ms',
outputvis=desel,
field='0,2,3,4,5,6',
spw='0',
datacolumn='all')
mslocal = mstool()
mslocal.open(outfile, nomodify=False)
mslocal.concatenate(msfile=desel)
mslocal.close()
ret = (verify_ms(outfile, 2, 32, 0))
self.assertTrue(ret[0], ret[1])
def setUp_vis_g(self):
if testmms:
os.system('cp -RL ' + datapath + vis_g + ' .')
elif(not os.path.exists(vis_g)):
# construct an MS with attached Jupiter ephemeris from vis_c
self.setUp_vis_c()
split(vis=vis_c, outputvis=vis_g, field='JUPITER', datacolumn='data')
mytb.open(vis_g, nomodify=False)
a = mytb.getcol('TIME')
delta = (54709.*86400-a[0])
a = a + delta
strt = a[0]
mytb.putcol('TIME', a)
a = mytb.getcol('TIME_CENTROID')
a = a + delta
mytb.putcol('TIME_CENTROID', a)
mytb.close()
mytb.open(vis_g+'/OBSERVATION', nomodify=False)
a = mytb.getcol('TIME_RANGE')
delta = strt - a[0][0]
a = a + delta
mytb.putcol('TIME_RANGE', a)
mytb.close()
mytb.open(vis_g+'/FIELD', nomodify=False)
a = mytb.getcol('TIME')
delta = strt - a[0]
a = a + delta
mytb.putcol('TIME', a)
mytb.close()
myms.open(vis_g, nomodify=False)
myms.addephemeris(0,os.environ.get('CASAPATH').split()[0]+'/data/ephemerides/JPL-Horizons/Jupiter_54708-55437dUTC.tab',
'Jupiter_54708-55437dUTC', 0)
myms.close()
def split_lambda():
if request.method == 'GET':
file = request.args.get('file')
elif request.method == 'POST':
if not (request.json):
abort(400)
file = request.json['file']
split(file)
return "Request Received"
def run_cal(cfg, section, vis, neb=None):
# Check for EB specific sections
if neb is not None:
aux = '%s%i' % (section, neb)
if cfg.has_section(aux):
sect = aux
ebind = None
else:
sect = section
ebind = neb - 1
else:
sect = section
ebind = None
# Calibrate
if not cfg.has_section(sect):
casalog.post('Apply (self-)calibration table not requested')
return
else:
# Back up flags
casalog.post('Using gaintable from section: %s' % sect)
flagmanager(vis=vis,
mode='save',
versionname='before_selfcal',
merge='replace',
comment='before selfcal')
# Applycal
field = cfg.get(sect, 'field')
gaintable = get_value(cfg, sect, 'gaintable', n=ebind).split()
spwmap = map(int, get_value(cfg, sect, 'spwmap', n=ebind).split())
calwt = get_bool(get_value(cfg, sect, 'calwt', n=ebind).split())
flagbackup = cfg.getboolean(sect, 'flagbackup')
interp = get_value(cfg, sect, 'interp', n=ebind, sep=';').split()
applycal(vis=vis,
field=field,
spwmap=spwmap,
gaintable=gaintable,
calwt=calwt,
flagbackup=flagbackup,
interp=interp)
# Split ms
outsplitvis = vis + '.selfcal'
split(vis=vis, outputvis=outsplitvis, datacolumn='corrected')
def setUp_vis_g(self):
if testmms:
os.system('cp -RL ' + datapath + vis_g + ' .')
elif (not os.path.exists(vis_g)):
# construct an MS with attached Jupiter ephemeris from vis_c
self.setUp_vis_c()
split(vis=vis_c,
outputvis=vis_g,
field='JUPITER',
datacolumn='data')
mytb.open(vis_g, nomodify=False)
a = mytb.getcol('TIME')
delta = (54709. * 86400 - a[0])
a = a + delta
strt = a[0]
mytb.putcol('TIME', a)
a = mytb.getcol('TIME_CENTROID')
a = a + delta
mytb.putcol('TIME_CENTROID', a)
mytb.close()
mytb.open(vis_g + '/OBSERVATION', nomodify=False)
a = mytb.getcol('TIME_RANGE')
delta = strt - a[0][0]
a = a + delta
mytb.putcol('TIME_RANGE', a)
mytb.close()
mytb.open(vis_g + '/FIELD', nomodify=False)
a = mytb.getcol('TIME')
delta = strt - a[0]
a = a + delta
mytb.putcol('TIME', a)
mytb.close()
myms.open(vis_g, nomodify=False)
myms.addephemeris(
0,
os.environ.get('CASAPATH').split()[0] +
'/data/ephemerides/JPL-Horizons/Jupiter_54708-55437dUTC.tab',
'Jupiter_54708-55437dUTC', 0)
myms.close()
field = basename.split("_")[0]
if exclude_7m:
arrayname = '12M'
if os.getenv("USE_SELFCAL_MS"):
antennae = ""
else:
msmd.open(continuum_ms)
antennae = ",".join([x for x in msmd.antennanames() if 'CM' not in x])
msmd.close()
# split out the 12M-only data to make further processing slightly
# faster
new_continuum_ms = continuum_ms.replace(".cal.ms", "_12M.cal.ms")
split(vis=continuum_ms, outputvis=new_continuum_ms, antenna=antennae,
field=field, datacolumn='data')
continuum_ms = new_continuum_ms
else:
antennae = ""
arrayname = '7M12M'
if os.getenv("USE_SELFCAL_MS"):
selfcal_ms = basename+"_"+arrayname+"_selfcal.ms"
continuum_ms = selfcal_ms
logprint("Imaging MS {0} with array {1}".format(continuum_ms, arrayname),
origin='almaimf_cont_imaging')
coosys,racen,deccen = determine_phasecenter(ms=continuum_ms, field=field)
phasecenter = "{0} {1}deg {2}deg".format(coosys, racen, deccen)
(dra,ddec,pixscale) = list(determine_imsize(ms=continuum_ms, field=field,
flag_filename = "{}_lines_flags.txt".format(parentdir)
# flag_path = os.path.expanduser("~/LocalGroup-VLA/15A-175/track_flagging")
flag_path = os.path.expanduser(
"~/ownCloud/code_development/LocalGroup-VLA/15A-175/track_flagging")
full_flag_filename = os.path.join(flag_path, flag_filename)
if os.path.exists(full_flag_filename):
copyfile(full_flag_filename,
os.path.join(lines_folder, "additional_flagging.txt"))
else:
print("No additional flagging script found in the VLA_Lband repo"
" for lines.")
split(vis=ms_active,
outputvis=lines_folder + "/" + mySDM + ".speclines.ms",
spw="8~12",
datacolumn='DATA',
field="")
# While it would be nice to remove the pol cal scans here, the pipeline
# will fail when running fluxboot because there is no other calibration
# field to transfer to! We can avoid this by just keeping all of the
# fields, even if they aren't used.
# field="0137+331=3C48,M33_2,M33_14,M33_6,M33_7_center,M33_12,M33_11,M33_8")
if split_cont:
cont_folder = parentdir + '_continuum'
if not os.path.exists(cont_folder):
os.mkdir(cont_folder)
else:
logprint(
"Splitting {0}'s spw {2} to {1}".format(
vis, outvis, spws[newid]), )
tb.open(invis)
if 'CORRECTED_DATA' in tb.colnames():
datacolumn = 'corrected'
else:
datacolumn = 'data'
tb.close()
assert split(
vis=invis,
spw=spws[newid],
field=field,
outputvis=outvis,
# there is no corrected_data column because we're
# splitting from split MSes
datacolumn=datacolumn,
)
if outvis in to_image[band][field][newid]:
raise ValueError()
to_image[band][field][newid].append(outvis)
with open('to_image.json', 'w') as fh:
json.dump(to_image, fh)
logprint("Completed line ms splitting. Moving on to continuum splitting")
def test4(self):
"""hanningsmooth - Test 4: Theoretical and calculated values should be the same for MMS-case"""
# Split the input to decrease the running time
split(self.msfile, outputvis="splithan.ms", scan="1,2", datacolumn="data")
self.msfile = "splithan.ms"
# create a test MMS. It creates self.testmms
self.createMMS(self.msfile)
self.outputms = "hann4.mms"
# check correct flagging (just for one row as a sample)
mslocal = mstool()
mslocal.open(self.msfile)
mslocal.sort(
"sorted.ms",
["OBSERVATION_ID", "ARRAY_ID", "SCAN_NUMBER", "FIELD_ID", "DATA_DESC_ID", "ANTENNA1", "ANTENNA2", "TIME"],
)
mslocal.close()
self.msfile = "sorted.ms"
flag_col = th.getVarCol(self.msfile, "FLAG")
self.assertTrue(flag_col["r1"][0][0] == [False])
self.assertTrue(flag_col["r1"][0][1] == [False])
self.assertTrue(flag_col["r1"][0][61] == [False])
self.assertTrue(flag_col["r1"][0][62] == [False])
data_col = th.getVarCol(self.msfile, "DATA")
hanningsmooth(vis=self.testmms, outputvis=self.outputms, datacolumn="data", keepmms=True)
self.assertTrue(ParallelDataHelper.isParallelMS(self.outputms), "Output should be an MMS")
# Sort the MMS
mslocal.open(self.outputms)
mslocal.sort(
"sorted.mms",
["OBSERVATION_ID", "ARRAY_ID", "SCAN_NUMBER", "FIELD_ID", "DATA_DESC_ID", "ANTENNA1", "ANTENNA2", "TIME"],
)
mslocal.close()
self.outputms = "sorted.mms"
corr_col = th.getVarCol(self.outputms, "DATA")
nrows = len(corr_col)
# check correct flagging (just for one row as a sample)
flag_col = th.getVarCol(self.outputms, "FLAG")
self.assertTrue(flag_col["r1"][0][0] == [True])
self.assertTrue(flag_col["r1"][0][1] == [False])
self.assertTrue(flag_col["r1"][0][61] == [False])
self.assertTrue(flag_col["r1"][0][62] == [True])
# Loop over every 2nd row,pol and get the data for each channel
max = 1e-05
for i in range(1, nrows, 2):
row = "r%s" % i
# polarization is 0-1
for pol in range(0, 2):
# array's channels is 0-63
for chan in range(1, 62):
# channels must start from second and end before the last
data = data_col[row][pol][chan]
dataB = data_col[row][pol][chan - 1]
dataA = data_col[row][pol][chan + 1]
Smoothed = th.calculateHanning(dataB, data, dataA)
CorData = corr_col[row][pol][chan]
# Check the difference
self.assertTrue(abs(CorData - Smoothed) < max)
def test4(self):
'''hanningsmooth - Test 4: Theoretical and calculated values should be the same for MMS-case'''
# Split the input to decrease the running time
split(self.msfile,
outputvis='splithan.ms',
scan='1,2',
datacolumn='data')
self.msfile = 'splithan.ms'
# create a test MMS. It creates self.testmms
self.createMMS(self.msfile)
self.outputms = 'hann4.mms'
# check correct flagging (just for one row as a sample)
mslocal = mstool()
mslocal.open(self.msfile)
mslocal.sort('sorted.ms', [
'OBSERVATION_ID', 'ARRAY_ID', 'SCAN_NUMBER', 'FIELD_ID',
'DATA_DESC_ID', 'ANTENNA1', 'ANTENNA2', 'TIME'
])
mslocal.close()
self.msfile = 'sorted.ms'
flag_col = th.getVarCol(self.msfile, 'FLAG')
self.assertTrue(flag_col['r1'][0][0] == [False])
self.assertTrue(flag_col['r1'][0][1] == [False])
self.assertTrue(flag_col['r1'][0][61] == [False])
self.assertTrue(flag_col['r1'][0][62] == [False])
data_col = th.getVarCol(self.msfile, 'DATA')
hanningsmooth(vis=self.testmms,
outputvis=self.outputms,
datacolumn='data',
keepmms=True)
self.assertTrue(ParallelDataHelper.isParallelMS(self.outputms),
'Output should be an MMS')
# Sort the MMS
mslocal.open(self.outputms)
mslocal.sort('sorted.mms', [
'OBSERVATION_ID', 'ARRAY_ID', 'SCAN_NUMBER', 'FIELD_ID',
'DATA_DESC_ID', 'ANTENNA1', 'ANTENNA2', 'TIME'
])
mslocal.close()
self.outputms = 'sorted.mms'
corr_col = th.getVarCol(self.outputms, 'DATA')
nrows = len(corr_col)
# check correct flagging (just for one row as a sample)
flag_col = th.getVarCol(self.outputms, 'FLAG')
self.assertTrue(flag_col['r1'][0][0] == [True])
self.assertTrue(flag_col['r1'][0][1] == [False])
self.assertTrue(flag_col['r1'][0][61] == [False])
self.assertTrue(flag_col['r1'][0][62] == [True])
# Loop over every 2nd row,pol and get the data for each channel
max = 1e-05
for i in range(1, nrows, 2):
row = 'r%s' % i
# polarization is 0-1
for pol in range(0, 2):
# array's channels is 0-63
for chan in range(1, 62):
# channels must start from second and end before the last
data = data_col[row][pol][chan]
dataB = data_col[row][pol][chan - 1]
dataA = data_col[row][pol][chan + 1]
Smoothed = th.calculateHanning(dataB, data, dataA)
CorData = corr_col[row][pol][chan]
# Check the difference
self.assertTrue(abs(CorData - Smoothed) < max)
]
msmd.close()
tb.open(continuum_ms)
if 'CORRECTED_DATA' in tb.colnames():
datacolumn = 'corrected'
else:
datacolumn = 'data'
tb.close()
split(
vis=continuum_ms,
outputvis=selfcal_ms,
datacolumn=datacolumn,
antenna=antennae,
spw=spwstr,
width=width,
field=field,
)
logprint("Selfcal MS is: "
"{0}".format(selfcal_ms),
origin='contim_selfcal')
coosys, racen, deccen = determine_phasecenter(ms=selfcal_ms, field=field)
phasecenter = "{0} {1}deg {2}deg".format(coosys, racen, deccen)
(dra, ddec, pixscale) = list(
determine_imsize(ms=selfcal_ms,
field=field,
phasecenter=(racen, deccen),
parentdir = os.getcwd().split("/")[-1]
if len(line_spws) == 0:
logprint("No line SPWs found.",
logfileout='logs/mixed_setup_split.log')
else:
lines_folder = parentdir + '_speclines'
if not os.path.exists(lines_folder):
os.mkdir(lines_folder)
line_fields = copy(fields)
line_fields = line_fields[line_fields != "3C138"]
line_fields = line_fields[line_fields != "J0319+4130"]
split(vis=ms_active,
outputvis=lines_folder + "/" + SDM_name + ".speclines.ms",
spw=",".join(line_spws), datacolumn='DATA',
field=",".join(line_fields))
line_run = True
line_settings = {"SDM_Name": SDM_name + ".speclines",
"scratch": False,
"myHanning": 'n',
"folder_name": lines_folder}
if len(cont_spws) == 0:
logprint("No continuum SPWs found.",
logfileout='logs/mixed_setup_split.log')
else:
cont_folder = parentdir + '_continuum'
if not os.path.exists(cont_folder):
def test4(self):
'''hanningsmooth2 - Test 4: Theoretical and calculated values should be the same for MMS-case'''
# Split the input to decrease the running time
split(self.msfile, outputvis='splithan.ms',scan='1,2',datacolumn='data')
self.msfile = 'splithan.ms'
# create a test MMS. It creates self.testmms
self.createMMS(self.msfile)
self.outputms = 'hann4.mms'
# check correct flagging (just for one row as a sample)
mslocal = mstool()
mslocal.open(self.msfile)
mslocal.sort('sorted.ms',['OBSERVATION_ID','ARRAY_ID','SCAN_NUMBER','FIELD_ID','DATA_DESC_ID','ANTENNA1','ANTENNA2','TIME'])
mslocal.close()
self.msfile = 'sorted.ms'
flag_col = th.getVarCol(self.msfile, 'FLAG')
self.assertTrue(flag_col['r1'][0][0] == [False])
self.assertTrue(flag_col['r1'][0][1] == [False])
self.assertTrue(flag_col['r1'][0][61] == [False])
self.assertTrue(flag_col['r1'][0][62] == [False])
data_col = th.getVarCol(self.msfile, 'DATA')
hanningsmooth2(vis=self.testmms, outputvis=self.outputms, datacolumn='data', keepmms=True)
self.assertTrue(ParallelDataHelper.isParallelMS(self.outputms), 'Output should be an MMS')
# Sort the MMS
mslocal.open(self.outputms)
mslocal.sort('sorted.mms',['OBSERVATION_ID','ARRAY_ID','SCAN_NUMBER','FIELD_ID','DATA_DESC_ID','ANTENNA1','ANTENNA2','TIME'])
mslocal.close()
self.outputms = 'sorted.mms'
corr_col = th.getVarCol(self.outputms, 'DATA')
nrows = len(corr_col)
# check correct flagging (just for one row as a sample)
flag_col = th.getVarCol(self.outputms, 'FLAG')
self.assertTrue(flag_col['r1'][0][0] == [True])
self.assertTrue(flag_col['r1'][0][1] == [False])
self.assertTrue(flag_col['r1'][0][61] == [False])
self.assertTrue(flag_col['r1'][0][62] == [True])
# Loop over every 2nd row,pol and get the data for each channel
max = 1e-05
for i in range(1,nrows,2) :
row = 'r%s'%i
# polarization is 0-1
for pol in range(0,2) :
# array's channels is 0-63
for chan in range(1,62) :
# channels must start from second and end before the last
data = data_col[row][pol][chan]
dataB = data_col[row][pol][chan-1]
dataA = data_col[row][pol][chan+1]
Smoothed = th.calculateHanning(dataB,data,dataA)
CorData = corr_col[row][pol][chan]
# Check the difference
self.assertTrue(abs(CorData-Smoothed) < max )
spw_num = spw_setup[line_name]['spw_num']
# Split out full SPW:
outputvis_name = "{0}/{1}/{2}_{3}.ms".format(
data_path, line_name,
os.path.split(ms_name)[1].rstrip(".ms"), line_name)
if os.path.exists(outputvis_name):
print("Found split MS for {0} {1}".format(
prefix, line_name))
else:
split(vis=ms_name,
outputvis=outputvis_name,
spw=spw_num,
intent='OBSERVE_TARGET#ON_SOURCE',
datacolumn='CORRECTED',
keepflags=True)
# Now continuum subtract with the line-free channels
linefree_chan = linefree_chans[line_name]
if len(linefree_chan) == 0:
print("No line free channels found for {0} {1}".format(
prefix, line_name))
linefree_str = ["{0}~{1}".format(*ch) for ch in linefree_chan]
if len(linefree_chans) > 1:
linefree_str = ";".join(linefree_str)
else:
def test_default(self):
'''Partition: create an MMS with default values in parallel'''
# First split off one scan to run the test faster
split(vis=self.msfile,
outputvis='split30.ms',
datacolumn='DATA',
scan='30')
msfile = 'split30.ms'
partition(vis=msfile, outputvis=self.mmsfile)
self.assertTrue(os.path.exists(self.mmsfile),
'MMS was not created for this test')
# Gather several metadata information
# for the MS
mdlocal1 = msmdtool()
mdlocal1.open(msfile)
ms_rows = mdlocal1.nrows()
ms_nscans = mdlocal1.nscans()
ms_nspws = mdlocal1.nspw()
ms_scans = mdlocal1.scannumbers()
mdlocal1.close()
# for the MMS
mdlocal2 = msmdtool()
mdlocal2.open(self.mmsfile)
mms_rows = mdlocal2.nrows()
mms_nscans = mdlocal2.nscans()
mms_nspws = mdlocal2.nspw()
mms_scans = mdlocal2.scannumbers()
mdlocal2.close()
# Compare the number of rows
self.assertEqual(ms_rows, mms_rows,
'Compare total number of rows in MS and MMS')
self.assertEqual(ms_nscans, mms_nscans, 'Compare number of scans')
self.assertEqual(ms_nspws, mms_nspws, 'Compare number of spws')
# Compare the scans
self.assertEqual(ms_scans.all(), mms_scans.all(),
'Compare all scan IDs')
try:
mdlocal1.open(msfile)
mdlocal2.open(self.mmsfile)
# Compare the spws
for i in ms_scans:
msi = mdlocal1.spwsforscan(i)
mmsi = mdlocal2.spwsforscan(i)
self.assertEqual(msi.all(), mmsi.all(),
'Compare spw Ids for a scan')
finally:
mdlocal1.close()
mdlocal2.close()
# Sort the output MSs so that they can be compared
myms = mstool()
myms.open(msfile)
myms.sort('ms_sorted.ms', [
'OBSERVATION_ID', 'ARRAY_ID', 'SCAN_NUMBER', 'FIELD_ID',
'DATA_DESC_ID', 'ANTENNA1', 'ANTENNA2', 'TIME'
])
myms.done()
myms.open(self.mmsfile)
myms.sort('mms_sorted.ms', [
'OBSERVATION_ID', 'ARRAY_ID', 'SCAN_NUMBER', 'FIELD_ID',
'DATA_DESC_ID', 'ANTENNA1', 'ANTENNA2', 'TIME'
])
myms.done()
# Ignore WEIGHT_SPECTRUM and SIGMA_SPECTRUM, which are empty columns
self.assertTrue(
th.compTables('ms_sorted.ms', 'mms_sorted.ms', [
'FLAG', 'FLAG_CATEGORY', 'TIME_CENTROID', 'WEIGHT_SPECTRUM',
'SIGMA_SPECTRUM', 'DATA'
]))
# Compare the DATA column
self.assertTrue(
th.compVarColTables('ms_sorted.ms', 'mms_sorted.ms', 'DATA'))
# The separation axis should be written to the output MMS
sepaxis = ph.axisType(self.mmsfile)
self.assertEqual(sepaxis, 'scan,spw',
'Partition did not write AxisType correctly in MMS')
def test_default(self):
"""Partition: create an MMS with default values in parallel"""
# First split off one scan to run the test faster
split(vis=self.msfile, outputvis="split30.ms", datacolumn="DATA", scan="30")
msfile = "split30.ms"
partition(vis=msfile, outputvis=self.mmsfile)
self.assertTrue(os.path.exists(self.mmsfile), "MMS was not created for this test")
# Gather several metadata information
# for the MS
mdlocal1 = msmdtool()
mdlocal1.open(msfile)
ms_rows = mdlocal1.nrows()
ms_nscans = mdlocal1.nscans()
ms_nspws = mdlocal1.nspw()
ms_scans = mdlocal1.scannumbers()
mdlocal1.close()
# for the MMS
mdlocal2 = msmdtool()
mdlocal2.open(self.mmsfile)
mms_rows = mdlocal2.nrows()
mms_nscans = mdlocal2.nscans()
mms_nspws = mdlocal2.nspw()
mms_scans = mdlocal2.scannumbers()
mdlocal2.close()
# Compare the number of rows
self.assertEqual(ms_rows, mms_rows, "Compare total number of rows in MS and MMS")
self.assertEqual(ms_nscans, mms_nscans, "Compare number of scans")
self.assertEqual(ms_nspws, mms_nspws, "Compare number of spws")
# Compare the scans
self.assertEqual(ms_scans.all(), mms_scans.all(), "Compare all scan IDs")
try:
mdlocal1.open(msfile)
mdlocal2.open(self.mmsfile)
# Compare the spws
for i in ms_scans:
msi = mdlocal1.spwsforscan(i)
mmsi = mdlocal2.spwsforscan(i)
self.assertEqual(msi.all(), mmsi.all(), "Compare spw Ids for a scan")
finally:
mdlocal1.close()
mdlocal2.close()
# Sort the output MSs so that they can be compared
myms = mstool()
myms.open(msfile)
myms.sort(
"ms_sorted.ms",
["OBSERVATION_ID", "ARRAY_ID", "SCAN_NUMBER", "FIELD_ID", "DATA_DESC_ID", "ANTENNA1", "ANTENNA2", "TIME"],
)
myms.done()
myms.open(self.mmsfile)
myms.sort(
"mms_sorted.ms",
["OBSERVATION_ID", "ARRAY_ID", "SCAN_NUMBER", "FIELD_ID", "DATA_DESC_ID", "ANTENNA1", "ANTENNA2", "TIME"],
)
myms.done()
self.assertTrue(
th.compTables(
"ms_sorted.ms", "mms_sorted.ms", ["FLAG", "FLAG_CATEGORY", "TIME_CENTROID", "WEIGHT_SPECTRUM", "DATA"]
)
)
# Compare the DATA column
self.assertTrue(th.compVarColTables("ms_sorted.ms", "mms_sorted.ms", "DATA"))
# The separation axis should be written to the output MMS
sepaxis = ph.axisType(self.mmsfile)
self.assertEqual(sepaxis, "scan,spw", "Partition did not write AxisType correctly in MMS")
def test_default(self):
'''Partition: create an MMS with default values in parallel'''
# First split off one scan to run the test faster
split(vis=self.msfile, outputvis='split30.ms', datacolumn='DATA', scan='30')
msfile = 'split30.ms'
partition(vis=msfile, outputvis=self.mmsfile)
self.assertTrue(os.path.exists(self.mmsfile), 'MMS was not created for this test')
# Gather several metadata information
# for the MS
mdlocal1 = msmdtool()
mdlocal1.open(msfile)
ms_rows = mdlocal1.nrows()
ms_nscans = mdlocal1.nscans()
ms_nspws = mdlocal1.nspw()
ms_scans = mdlocal1.scannumbers()
mdlocal1.close()
# for the MMS
mdlocal2 = msmdtool()
mdlocal2.open(self.mmsfile)
mms_rows = mdlocal2.nrows()
mms_nscans = mdlocal2.nscans()
mms_nspws = mdlocal2.nspw()
mms_scans = mdlocal2.scannumbers()
mdlocal2.close()
# Compare the number of rows
self.assertEqual(ms_rows, mms_rows, 'Compare total number of rows in MS and MMS')
self.assertEqual(ms_nscans, mms_nscans, 'Compare number of scans')
self.assertEqual(ms_nspws, mms_nspws, 'Compare number of spws')
# Compare the scans
self.assertEqual(ms_scans.all(), mms_scans.all(), 'Compare all scan IDs')
try:
mdlocal1.open(msfile)
mdlocal2.open(self.mmsfile)
# Compare the spws
for i in ms_scans:
msi = mdlocal1.spwsforscan(i)
mmsi = mdlocal2.spwsforscan(i)
self.assertEqual(msi.all(), mmsi.all(), 'Compare spw Ids for a scan')
finally:
mdlocal1.close()
mdlocal2.close()
# Sort the output MSs so that they can be compared
myms = mstool()
myms.open(msfile)
myms.sort('ms_sorted.ms',['OBSERVATION_ID','ARRAY_ID','SCAN_NUMBER','FIELD_ID','DATA_DESC_ID','ANTENNA1','ANTENNA2','TIME'])
myms.done()
myms.open(self.mmsfile)
myms.sort('mms_sorted.ms',['OBSERVATION_ID','ARRAY_ID','SCAN_NUMBER','FIELD_ID','DATA_DESC_ID','ANTENNA1','ANTENNA2','TIME'])
myms.done()
# Ignore WEIGHT_SPECTRUM and SIGMA_SPECTRUM, which are empty columns
self.assertTrue(th.compTables('ms_sorted.ms', 'mms_sorted.ms',
['FLAG','FLAG_CATEGORY','TIME_CENTROID',
'WEIGHT_SPECTRUM','SIGMA_SPECTRUM','DATA']))
# Compare the DATA column
self.assertTrue(th.compVarColTables('ms_sorted.ms', 'mms_sorted.ms','DATA'))
# The separation axis should be written to the output MMS
sepaxis = ph.axisType(self.mmsfile)
self.assertEqual(sepaxis, 'scan,spw', 'Partition did not write AxisType correctly in MMS')
flagdata(vis=visfile, mode='manual', spw=linechannels,
flagbackup=False)
flagmanager(vis=visfile, mode='save',
versionname='line_channel_flags')
rmtables(contvis)
os.system('rm -rf ' + contvis + '.flagversions')
# Average the channels within spws
# (assert here checks that this completes successfully)
rslt = split(vis=visfile,
spw=",".join(map(str, spws)),
field=field,
outputvis=contvis,
width=widths,
datacolumn=datacolumn)
print("split's result was {0}".format(rslt))
if not os.path.exists(contvis):
raise IOError("Split failed for {0}".format(contvis))
# If you flagged any line channels, restore the previous flags
flagmanager(vis=visfile, mode='restore',
versionname='before_cont_flags')
# flag out the autocorres
flagdata(vis=contvis, mode='manual', autocorr=True)
# I revised this later because it appears that at least one
# window legitimately had edge channels flagged out
# (check_channel_flags will raise an exception if there is excess flagging)
# Also, Luke reported that there are some cases in which chunks were
# flagged out because of bad atmospheric absorption lines in part of the band;
# we think this only affected 7m data?
check_channel_flags(invis,
field=field,
spw=str(spws[newid]),
tolerance=0.1)
result = split(
vis=invis,
spw=spws[newid],
field=field,
outputvis=outvis,
# there is no corrected_data column because we're
# splitting from split MSes
datacolumn=datacolumn,
)
print("Split ended with result={0} in line split".format(
result))
flagdata(vis=outvis, mode='manual', autocorr=True)
if outvis in to_image[band][field][newid]:
raise ValueError()
to_image[band][field][newid].append(outvis)
with open('to_image.json', 'w') as fh:
fitspw='8:1024~3072',
excludechans=True,
want_cont=False,
fitorder=1,
spw='8')
msforsplit = msname + '.contsub'
dataforsplit = 'data'
spwsplit = ''
#
# Split
#
everythings_log("Start final split")
mssplit = msname + '.split'
split(vis=msforsplit,
outputvis=mssplit,
spw=spwsplit,
intent='*TARGET*',
width=4,
datacolumn=dataforsplit)
# Set spw name to avoid further warning
temp = vishead(mssplit, mode='get', hdkey='spw_name')
temp = list(temp[0][0])
temp[-1] = '0'
temp = ''.join(temp)
vishead(mssplit, mode='put', hdkey='spw_name', hdvalue=temp)
# Remove contsub file
if contsub:
rmtree(msforsplit)
#
# Tar original ms
#
if tar_ms:
