def test_mms_XXYY_selection(self):
'''mstransform: correlation='RR,LL' should select and re-index properly'''
self.outputms = '3cRRLL.mms'
# spw 0 should not be processed. The selection should happen before the MMS work
mstransform(vis=self.vis, outputvis=self.outputms, datacolumn='data', correlation='RR,LL',
createmms=True, separationaxis='auto')
msmdt = msmdtool()
msmdt.open(self.outputms)
out_dds = msmdt.datadescids()
msmdt.done()
ref = [0,1]
for i in out_dds:
self.assertEqual(out_dds[i], ref[i])
pol_col = th.getVarCol(self.outputms+'/POLARIZATION','NUM_CORR')
self.assertEqual(pol_col['r1'][0], 0,'Error in NUM_CORR of POLARIZATION table')
self.assertEqual(pol_col['r2'][0], 0,'Error in NUM_CORR of POLARIZATION table')
self.assertEqual(pol_col['r3'][0], 2,'Error in NUM_CORR of POLARIZATION table')
self.assertEqual(pol_col['r4'][0], 2,'Error in NUM_CORR of POLARIZATION table')
# Verify that POLARIZATION table is not re-sized.
corr_col = th.getVarCol(self.outputms+'/POLARIZATION', 'NUM_CORR')
self.assertEqual(corr_col.keys().__len__(), 4, 'Wrong number of rows in POLARIZATION table')
# Check the FEED table
# out_feed_spw = th.getVarCol(self.outputms+'/FEED', 'SPECTRAL_WINDOW_ID')
# self.assertEqual(len(out_feed_spw.keys()), 52)
# listobs, listpartition should not fail
listobs(self.outputms, listfile='3c_1.obs')
self.assertTrue(os.path.exists('3c_1.obs'), 'Probable error in sub-table re-indexing')
listpartition(self.outputms, listfile='3c_2.obs')
self.assertTrue(os.path.exists('3c_2.obs'), 'Probable error in sub-table re-indexing')
def test_split_MMS(self):
'''mstransform: Split MMS in parallel'''
# Create an MMS in the setup. It creates self.testmms
self.createMMS(self.vis, axis='scan', spws='0,1')
self.outputms = 'scan30.mms'
mstransform(vis=self.testmms, outputvis=self.outputms, datacolumn='data', scan='30')
self.assertTrue(ParallelTaskHelper.isParallelMS(self.outputms),'Output is not an MMS')
mslocal = mstool()
mslocal.open(self.outputms)
sublist = mslocal.getreferencedtables()
self.assertEqual(len(sublist), 1)
# Test DD table
msmdt = msmdtool()
msmdt.open(self.outputms)
out_dds = msmdt.datadescids()
msmdt.done()
ref = [0,1]
for i in out_dds:
self.assertEqual(out_dds[i], ref[i])
# The separation axis should be copied to the output MMS
in_sepaxis = ph.axisType(self.testmms)
out_sepaxis = ph.axisType(self.outputms)
self.assertEqual(in_sepaxis, out_sepaxis, 'AxisTypes from input and output MMS do not match')
def test_mms_scan_spw_partition(self):
'''mstransform: Create MMS and part by scan/spw'''
self.outputms = '3cscanspw02.mms'
mstransform(vis=self.vis, outputvis=self.outputms, datacolumn='data', spw='0,2',
createmms=True, disableparallel=True, separationaxis='auto')
# Verify the input versus the output
msmdt = msmdtool()
msmdt.open(self.outputms)
out_dds = msmdt.datadescids()
msmdt.done()
ref = [0,1,2]
for i in out_dds:
self.assertEqual(out_dds[i], ref[i])
# Verify that DATA_DESCRIPTION table is properly re-indexed.
spw_col = th.getVarCol(self.outputms+'/DATA_DESCRIPTION', 'SPECTRAL_WINDOW_ID')
self.assertEqual(spw_col.keys().__len__(), 3, 'Wrong number of rows in DD table')
self.assertEqual(spw_col['r1'][0], 0,'Error re-indexing SPECTRAL_WINDOW_ID of DATA_DESCRIPTION table')
self.assertEqual(spw_col['r2'][0], 0,'Error re-indexing SPECTRAL_WINDOW_ID of DATA_DESCRIPTION table')
self.assertEqual(spw_col['r3'][0], 1,'Error re-indexing SPECTRAL_WINDOW_ID of DATA_DESCRIPTION table')
# Check the FEED table
out_feed_spw = th.getVarCol(self.outputms+'/FEED', 'SPECTRAL_WINDOW_ID')
self.assertEqual(out_feed_spw['r1'],[0])
self.assertEqual(out_feed_spw['r26'],[0])
self.assertEqual(out_feed_spw['r27'],[1])
self.assertEqual(out_feed_spw['r28'],[1])
self.assertEqual(out_feed_spw['r51'],[1])
self.assertEqual(len(out_feed_spw.keys()), 52)
def test_split_MMS_weight_corr_sel(self):
'''mstransform: Split MMS in parallel. Check WEIGHT shape when selecting correlation'''
# Create an MMS in the setup. It creates self.testmms
self.createMMS(self.vis, axis='scan', spws='0,1')
self.outputms = 'corrRR_LL.mms'
mstransform(vis=self.testmms, outputvis=self.outputms, datacolumn='data', correlation='RR,LL',spw='0')
self.assertTrue(ParallelTaskHelper.isParallelMS(self.outputms),'Output is not an MMS')
mslocal = mstool()
mslocal.open(self.outputms)
sublist = mslocal.getreferencedtables()
self.assertEqual(len(sublist), 2)
# Test DD table
msmdt = msmdtool()
msmdt.open(self.outputms)
out_dds = msmdt.datadescids()
msmdt.done()
ref = [0]
for i in out_dds:
self.assertEqual(out_dds[i], ref[i])
# The separation axis should be copied to the output MMS
in_sepaxis = ph.axisType(self.testmms)
out_sepaxis = ph.axisType(self.outputms)
self.assertEqual(in_sepaxis, out_sepaxis, 'AxisTypes from input and output MMS do not match')
# Check the dimensions of the WEIGHT and SIGMA columns. CAS-6946
out_ws = th.getColShape(self.outputms,'WEIGHT')
out_ss = th.getColShape(self.outputms,'SIGMA')
self.assertEqual(out_ws[0],'[2]','WEIGHT shape is not correct')
self.assertEqual(out_ss[0],'[2]','SIGMA shape is not correct')
def getPlotantsObservatoryInfo(msname): metadata = msmdtool() metadata.open(msname) telescope = metadata.observatorynames()[0] arrayPos = metadata.observatoryposition() metadata.close() return telescope, arrayPos
def test_float_data_mms_baseline_auto(self):
'''importasdm: Create an MMS with a FLOAT_DATA column separated per baseline '''
myasdmname = 'uid___A002_X6218fb_X264'
themsname = myasdmname + ".ms"
# The ocorr_mode='ao' option will create a FLOAT_DATA column instead of DATA
importasdm(myasdmname,
vis=themsname,
ocorr_mode='ao',
createmms=True,
scans='1',
separationaxis='baseline',
numsubms=4)
self.assertTrue(ParallelDataHelper.isParallelMS(themsname),
'Output is not a Multi-MS')
self.assertTrue(len(th.getColDesc(themsname, 'FLOAT_DATA')) > 0)
md = msmdtool()
md.open(themsname)
bsl = md.baselines()
md.close()
#diagnoals give the auto-corrs
ac = bsl.diagonal()
self.assertTrue(ac.all(), 'Not all auto-correlations are there')
# Take the dictionary and compare with original MS
thisdict = listpartition(vis=themsname, createdict=True)
self.assertEqual(len(thisdict.keys()), 4,
'There should be 4 subMSs in output MMS')
def test_mms_spw_selection3(self):
'''mstransform: Create MMS and select three spws with numsubms=2'''
self.outputms = '3cspw012.mms'
mstransform(vis=self.vis, outputvis=self.outputms, datacolumn='data', spw='0,1,2',
createmms=True, separationaxis='spw', numsubms=2)
# Verify the input versus the output
msmdt = msmdtool()
msmdt.open(self.outputms)
out_dds = msmdt.datadescids()
out_nrow = msmdt.nrows()
msmdt.done()
self.assertTrue(out_nrow,5200)
ref = [0,1,2,3]
for i in out_dds:
self.assertEqual(out_dds[i], ref[i])
# Verify that DATA_DESCRIPTION table is properly re-indexed.
spw_col = th.getVarCol(self.outputms+'/DATA_DESCRIPTION', 'SPECTRAL_WINDOW_ID')
self.assertEqual(spw_col.keys().__len__(), 4, 'Wrong number of rows in DD table')
self.assertEqual(spw_col['r1'][0], 0,'Error re-indexing SPECTRAL_WINDOW_ID of DATA_DESCRIPTION table')
self.assertEqual(spw_col['r2'][0], 0,'Error re-indexing SPECTRAL_WINDOW_ID of DATA_DESCRIPTION table')
self.assertEqual(spw_col['r3'][0], 1,'Error re-indexing SPECTRAL_WINDOW_ID of DATA_DESCRIPTION table')
self.assertEqual(spw_col['r4'][0], 2,'Error re-indexing SPECTRAL_WINDOW_ID of DATA_DESCRIPTION table')
in_feed_tb = th.getVarCol(self.vis+'/FEED', 'SPECTRAL_WINDOW_ID')
out_feed_tb = th.getVarCol(self.outputms+'/FEED', 'SPECTRAL_WINDOW_ID')
# Check the FEED table
th.compTables(self.vis+'/FEED', self.outputms+'/FEED', ['FOCUS_LENGTH'])
def freq_selection_overlap(ms, freqsel, spw=0):
"""
For a given frequency selection string (e.g., '215~216GHz;900~950GHz'),
find the subset of the `;`-separated entries that overlap with the
measurement set and return those.
Parameters
----------
ms : str
The name of a measurement set to compare against
freqsel : str
A frequency-based selection string
spw : int
The spectral window number, default to 0
"""
# use the default (topo or lsrk or whatever) frequency, don't cvel it,
# because the inputs are assumed to be in the same frame.
# it's important to be exact here, since we're using the fmin/fmax
# to fill in when one of the ends of a selection is out of range.
msmd = msmdtool()
msmd.open(ms)
frequencies_in_ms = msmd.chanfreqs(spw)
msmd.close()
fmin, fmax = frequencies_in_ms.min(), frequencies_in_ms.max()
new_sel = []
for selstr in freqsel.split(";"):
lo, hi = map(float, selstr.strip(string.ascii_letters).split("~"))
unit = selstr.lstrip(string.punctuation + string.digits)
if lo > hi:
lo, hi = hi, lo
flo = qq.convert({'value': float(lo), 'unit': unit}, 'Hz')['value']
fhi = qq.convert({'value': float(hi), 'unit': unit}, 'Hz')['value']
if ((fhi < fmax) and (fhi > fmin)) and ((flo > fmin) and
(flo < fmax)):
# if the whole thing is in range...
new_sel.append(selstr)
elif ((fhi < fmax) and (fhi > fmin)) and not ((flo > fmin) and
(flo < fmax)):
# if the lower end is out of range, but the upper is in
new_selstr = "{0}~{1}GHz".format(fmin / 1e9, fhi / 1e9)
new_sel.append(new_selstr)
elif (not ((fhi < fmax) and (fhi > fmin))) and ((flo > fmin) and
(flo < fmax)):
# if the upper end is out of range, but the lower is in
new_selstr = "{0}~{1}GHz".format(flo / 1e9, fmax / 1e9)
new_sel.append(new_selstr)
return "{0}:".format(spw) + ";".join(new_sel)
def test_baseline_autocorr(self):
'''partition: create an MMS per baseline axis only for auto-correlations'''
self.outputms = 'baseline_autocorr.mms'
partition(self.vis, outputvis=self.outputms, separationaxis='baseline', antenna='*&&&', flagbackup=False)
md = msmdtool()
md.open(self.outputms)
bsl = md.baselines()
md.close()
#diagnoals give the auto-corrs
ac = bsl.diagonal()
self.assertTrue(ac.all(), 'Not all auto-correlations are there')
def test_combspws_timespan_spw_axis_error(self):
'''mstransform: combinespws=True, timespan=scan axis=spw'''
self.createMMS(self.vis, axis='spw',scans='30',spws='10,11')
self.outputms = "spwaxiserror.mms"
# subMSs do not have all spws. Create an MS
mstransform(vis=self.testmms, outputvis=self.outputms, datacolumn='data',
combinespws=True, timeaverage=True, timebin='20s',timespan='scan')
self.assertFalse(ParallelDataHelper.isParallelMS(self.outputms),'Output should be an MS')
mymsmd = msmdtool()
mymsmd.open(self.outputms)
nspw = mymsmd.nspw()
mymsmd.close()
self.assertEqual(nspw,1)
def test_combspws_timespan_error(self):
'''mstransform: combinespws=True, timespan=scan axis=auto timebin=40s'''
self.createMMS(self.vis, axis='auto',spws='1,3', numms=4)
self.outputms = "spanscan_comb.mms"
# combinespws is not possible. It should create and MS
mstransform(vis=self.testmms, outputvis=self.outputms, datacolumn='data',
combinespws=True, timeaverage=True, timebin='40s',timespan='scan')
self.assertFalse(ParallelDataHelper.isParallelMS(self.outputms),'Output should be an MS')
mymsmd = msmdtool()
mymsmd.open(self.outputms)
nspw = mymsmd.nspw()
mymsmd.close()
self.assertEqual(nspw,1)
def test_combspws_timespan_fail(self):
'''mstransform: combinespws=True, timespan=scan axis=auto timebin=200s'''
self.createMMS(self.vis, axis='auto',spws='3')
self.outputms = "errormms.mms"
# Scans are shorter than timebin. Create an MS
mstransform(vis=self.testmms, outputvis=self.outputms, datacolumn='data',
combinespws=True, timeaverage=True, timebin='200s',timespan='scan')
self.assertFalse(ParallelDataHelper.isParallelMS(self.outputms),'Output should be an MS')
mymsmd = msmdtool()
mymsmd.open(self.outputms)
nscan = mymsmd.nscans()
exposure = mymsmd.exposuretime(30)['value']
mymsmd.close()
self.assertEqual(nscan,1)
self.assertEqual(exposure, 179)
def test_timespan_spw_axis(self):
'''mstransform: timeaverage=True, timespan=scan, separationaxis=spw'''
self.createMMS(self.vis, axis='spw',spws='1,3')
self.outputms = "spanscan_spw.mms"
mstransform(vis=self.testmms, outputvis=self.outputms, datacolumn='data',timeaverage=True,
timebin='100s',timespan='scan')
self.assertTrue(ParallelDataHelper.isParallelMS(self.outputms),'Output should be an MMS')
self.assertEqual(ph.axisType(self.outputms),'spw')
mymsmd = msmdtool()
mymsmd.open(self.outputms)
t30 = mymsmd.exposuretime(30)['value']
t31 = mymsmd.exposuretime(31)['value']
mymsmd.close()
self.assertEqual(t30, 100)
self.assertEqual(t31, 79)
def freq_selection_overlap(ms, freqsel, spw=0):
"""
For a given frequency selection string (e.g., '215~216GHz;900~950GHz'),
find the subset of the `;`-separated entries that overlap with the
measurement set and return those.
Parameters
----------
ms : str
The name of a measurement set to compare against
freqsel : str
A frequency-based selection string
spw : int
The spectral window number, default to 0
"""
msmd = msmdtool()
msmd.open(ms)
frequencies_in_ms = msmd.chanfreqs(spw)
msmd.close()
fmin, fmax = frequencies_in_ms.min(), frequencies_in_ms.max()
new_sel = []
for selstr in freqsel.split(";"):
lo, hi = map(float, selstr.strip(string.ascii_letters).split("~"))
unit = selstr.lstrip(string.punctuation + string.digits)
if lo > hi:
lo, hi = hi, lo
flo = qq.convert({'value': float(lo), 'unit': unit}, 'Hz')['value']
fhi = qq.convert({'value': float(hi), 'unit': unit}, 'Hz')['value']
if ((fhi < fmax) and (fhi > fmin)) and ((flo > fmin) and (flo < fmax)):
new_sel.append(selstr)
elif ((fhi < fmax) and (fhi > fmin)) and not ((flo > fmin) and
(flo < fmax)):
new_selstr = "{0}~{1}GHz".format(fmin / 1e9, fhi / 1e9)
new_sel.append(new_selstr)
elif (not ((fhi < fmax) and (fhi > fmin))) and ((flo > fmin) and
(flo < fmax)):
new_selstr = "{0}~{1}GHz".format(flo / 1e9, fmax / 1e9)
new_sel.append(new_selstr)
return "{0}:".format(spw) + ";".join(new_sel)
def test_sd_data_mms_baseline_all(self):
'''importasdm: Create an MMS separated per baseline, using default numsubms '''
myasdmname = 'uid___A002_X6218fb_X264'
themsname = myasdmname+".ms"
# Create a single-dish MMS with auto and cross correlations
importasdm(myasdmname, vis=themsname, ocorr_mode='ca', createmms=True, scans='1', separationaxis='baseline')
self.assertTrue(ParallelDataHelper.isParallelMS(themsname), 'Output is not a Multi-MS')
self.assertTrue(len(th.getColDesc(themsname, 'DATA')) > 0)
md = msmdtool()
md.open(themsname)
bsl = md.baselines()
md.close()
# Check if all baselines are in there
self.assertTrue(bsl.all(), 'Not all baselines are in the MMS')
def test_baseline_autocorr(self):
'''partition: create an MMS per baseline axis only for auto-correlations'''
self.outputms = 'baseline_autocorr.mms'
partition(self.vis,
outputvis=self.outputms,
separationaxis='baseline',
antenna='*&&&',
flagbackup=False)
md = msmdtool()
md.open(self.outputms)
bsl = md.baselines()
md.close()
#diagnoals give the auto-corrs
ac = bsl.diagonal()
self.assertTrue(ac.all(), 'Not all auto-correlations are there')
def test_baseline_partition(self):
'''partition: create an MMS per baseline for an interferometry MS'''
partition(vis=self.msfile, outputvis=self.mmsfile, spw='0,1',createmms=True,
separationaxis='baseline', flagbackup=False, datacolumn='data')
listpartition(self.mmsfile)
# This MS doesn't have auto-correlations. It has 4 antennas and 6 baselines (cross)
# There will be MSSelectionNullSelection SEVERE messages in the logfile because
# it will try to create SubMSs from auto-correlations. This is not an error!
md = msmdtool()
md.open(self.mmsfile)
bsl = md.baselines()
md.close()
# diagonal give the auto-corrs
ac = bsl.diagonal()
self.assertFalse(ac.all(), 'There should be no auto-correlations in MMS')
def test_float_data_mms_baseline_auto(self):
'''importasdm: Create an MMS with a FLOAT_DATA column separated per baseline '''
myasdmname = 'uid___A002_X6218fb_X264'
themsname = myasdmname+".ms"
# The ocorr_mode='ao' option will create a FLOAT_DATA column instead of DATA
importasdm(myasdmname, vis=themsname, ocorr_mode='ao', createmms=True, scans='1', separationaxis='baseline', numsubms=4)
self.assertTrue(ParallelDataHelper.isParallelMS(themsname), 'Output is not a Multi-MS')
self.assertTrue(len(th.getColDesc(themsname, 'FLOAT_DATA')) > 0)
md = msmdtool()
md.open(themsname)
bsl = md.baselines()
md.close()
#diagnoals give the auto-corrs
ac = bsl.diagonal()
self.assertTrue(ac.all(), 'Not all auto-correlations are there')
# Take the dictionary and compare with original MS
thisdict = listpartition(vis=themsname, createdict=True)
self.assertEqual(len(thisdict.keys()), 4, 'There should be 4 subMSs in output MMS')
def test_sd_data_mms_baseline_all(self):
'''importasdm: Create an MMS separated per baseline, using default numsubms '''
myasdmname = 'uid___A002_X6218fb_X264'
themsname = myasdmname + ".ms"
# Create a single-dish MMS with auto and cross correlations
importasdm(myasdmname,
vis=themsname,
ocorr_mode='ca',
createmms=True,
scans='1',
separationaxis='baseline')
self.assertTrue(ParallelDataHelper.isParallelMS(themsname),
'Output is not a Multi-MS')
self.assertTrue(len(th.getColDesc(themsname, 'DATA')) > 0)
md = msmdtool()
md.open(themsname)
bsl = md.baselines()
md.close()
# Check if all baselines are in there
self.assertTrue(bsl.all(), 'Not all baselines are in the MMS')
def test_baseline_partition(self):
'''partition: create an MMS per baseline for an interferometry MS'''
partition(vis=self.msfile,
outputvis=self.mmsfile,
spw='0,1',
createmms=True,
separationaxis='baseline',
flagbackup=False,
datacolumn='data')
listpartition(self.mmsfile)
# This MS doesn't have auto-correlations. It has 4 antennas and 6 baselines (cross)
# There will be MSSelectionNullSelection SEVERE messages in the logfile because
# it will try to create SubMSs from auto-correlations. This is not an error!
md = msmdtool()
md.open(self.mmsfile)
bsl = md.baselines()
md.close()
# diagonal give the auto-corrs
ac = bsl.diagonal()
self.assertFalse(ac.all(),
'There should be no auto-correlations in MMS')
if sys.version_info.major > 2:
from casatools import ms, quanta, msmetadata
from casatasks import casalog
casalog.showconsole(True)
datams = ms()
ms_in = ms()
datamsmd = msmetadata()
qa = quanta()
else:
from taskinit import ms, qa, mstool, msmdtool, casalog
datams = mstool()
ms_in = mstool()
datamsmd = msmdtool()
# from taskinit import *
# from callibrary import *
# import pdb
def subvs2(vis=None,
outputvis=None,
timerange='',
spw='',
mode='linear',
subtime1='',
subtime2='',
smoothaxis='time',
smoothtype='flat',
if os.getenv('ALMAIMF_ROOTDIR') is None:
try:
import metadata_tools
os.environ['ALMAIMF_ROOTDIR'] = os.path.split(
metadata_tools.__file__)[0]
except ImportError:
raise ValueError("metadata_tools not found on path; make sure to "
"specify ALMAIMF_ROOTDIR environment variable "
"or your PYTHONPATH variable to include the directory"
" containing the ALMAIMF code.")
else:
sys.path.append(os.getenv('ALMAIMF_ROOTDIR'))
from metadata_tools import check_channel_flags
msmd = msmdtool()
ms = mstool()
tb = tbtool()
# band name : frequency range (GHz)
bands = {
'B3': (80, 110),
'B6': (210, 250),
}
def logprint(string):
casalog.post(string, origin='split_line_windows')
print(string)
def genImageName(vis='', spw='', field='', imtype='mfs', targettype='sci', stokes='I', mous='', modtext='manual'):
# DPetry, Sept 2017
"""
Generate image name to be used in clean command
following the ALMA archive conventions (see SCIREQ-110, draft recommendations v4.8).
vis = MS to be imaged (to extract object name, SPW frequencies
spw = ID of SPW to be imaged
If several SPWs are imaged at once, enter list of SPWs.
field = ID of field to be imaged (to extract object name).
in case of mosaic, give any ID of field belonging to the mosaic
imtype = image type, 'mfs' (default) - aggregate bandwidth
'cont' - mfs with lines excluded
'cube' - spectrally resolved cube
targettype = 'sci' (default) for science target
'bp' for bandpass calibrator
'ph' for phase calibrator
'chk' for checksource
'amp' for flux calibrator
stokes = 'I' (default) for Stokes I image
any combination of 'I', 'Q', 'U', 'V', 'P', 'A'
mous = '' (default) normally this is filled by the archive.
But if you know the MOUS UID, enter it in the format with "_"
e.g. uid___A001_X888_X66
modtext = 'manual' (default) - this string can be used to add text,
i.e. a modification, to the standard file name.
It is appended at the end of the name.
"""
themous = ''
theobject = ''
thetargettype = ''
targettypes = ['sci', 'bp', 'ph', 'chk', 'amp']
thespwid = ''
theimtype = ''
imtypes = ['mfs', 'cont', 'cube']
thestokes = ''
stokess = ['I', 'Q', 'U', 'V', 'IQU', 'IQUV', 'P', 'A']
themodtext = ''
rval = ''
mymsmd = taskinit.msmdtool()
# MOUS
if (mous != ''):
themous = mous+'.'
# object
try:
mymsmd.open(vis)
except:
mymsmd.done()
raise Exception("ERROR: problem opening vis")
myfieldnames = mymsmd.fieldnames()
if type(field)==str and (field in myfieldnames):
theobject = field
elif field in range(mymsmd.nfields()):
theobject = myfieldnames[field]
else:
print "ERROR: invalid field: "+field
print "Valid entries are ", myfieldnames
print " or ", range(mymsmd.nfields())
raise Exception("ERROR: invalid field: "+field)
myspws = mymsmd.spwsforfield(theobject)
mymsmd.close()
# target type
if targettype in targettypes:
thetargettype = targettype
else:
print "ERROR: invalid targettype ", targettype
print " Valid entries are ", targettypes
raise Exception("ERROR: invalid targettype ", targettype)
# spw
if type(spw) != type([]):
spw = [spw]
for myspw in spw:
if myspw in myspws:
if thespwid=='':
thespwid = str(myspw)
else:
thespwid = thespwid+'_'+str(myspw)
else:
if not type(myspw) == int:
print "ERROR: invalid spw: "+str(myspw)+". Valid entries are ", myspws
raise Exception("ERROR: invalid spw: "+str(myspw)+' Data type must be int.')
else:
print "ERROR: invalid spw: "+str(myspw)+". Valid entries are ", myspws
raise Exception("ERROR: invalid spw: "+str(myspw))
# imtype
if imtype in imtypes:
theimtype = imtype
else:
print "ERROR: invalid imtype ", imtype
print " Valid entries are ", imtypes
raise Exception( "ERROR: invalid imtype "+str(imtype))
# stokes
if stokes in stokess:
thestokes = stokes
else:
print "ERROR: invalid stokes ", stokes
print " Valid entries are ", stokess
raise Exception("ERROR: invalid stokes "+str(stokes))
# modifying text
if modtext != '':
themodtext = '.'+modtext
if ' ' in themodtext:
raise Exception("ERROR: modtext must not contain spaces")
rval = themous+theobject+'_'+thetargettype+'.spw'+thespwid+'.'+theimtype+'.'+thestokes+themodtext
if ' ' in rval:
raise Exception("ERROR: generated name contains spaces: \""+rval+"\"")
if '/' in rval:
raise Exception("ERROR: generated name contains slash: \""+rval+"\"")
if '*' in rval:
raise Exception("ERROR: generated name contains star: \""+rval+"\"")
if '?' in rval:
raise Exception("ERROR: generated name contains question mark: \""+rval+"\"")
return rval
def test_default(self):
'''Partition: create an MMS with default values in parallel'''
# First split off one scan to run the test faster
split(vis=self.msfile,
outputvis='split30.ms',
datacolumn='DATA',
scan='30')
msfile = 'split30.ms'
partition(vis=msfile, outputvis=self.mmsfile)
self.assertTrue(os.path.exists(self.mmsfile),
'MMS was not created for this test')
# Gather several metadata information
# for the MS
mdlocal1 = msmdtool()
mdlocal1.open(msfile)
ms_rows = mdlocal1.nrows()
ms_nscans = mdlocal1.nscans()
ms_nspws = mdlocal1.nspw()
ms_scans = mdlocal1.scannumbers()
mdlocal1.close()
# for the MMS
mdlocal2 = msmdtool()
mdlocal2.open(self.mmsfile)
mms_rows = mdlocal2.nrows()
mms_nscans = mdlocal2.nscans()
mms_nspws = mdlocal2.nspw()
mms_scans = mdlocal2.scannumbers()
mdlocal2.close()
# Compare the number of rows
self.assertEqual(ms_rows, mms_rows,
'Compare total number of rows in MS and MMS')
self.assertEqual(ms_nscans, mms_nscans, 'Compare number of scans')
self.assertEqual(ms_nspws, mms_nspws, 'Compare number of spws')
# Compare the scans
self.assertEqual(ms_scans.all(), mms_scans.all(),
'Compare all scan IDs')
try:
mdlocal1.open(msfile)
mdlocal2.open(self.mmsfile)
# Compare the spws
for i in ms_scans:
msi = mdlocal1.spwsforscan(i)
mmsi = mdlocal2.spwsforscan(i)
self.assertEqual(msi.all(), mmsi.all(),
'Compare spw Ids for a scan')
finally:
mdlocal1.close()
mdlocal2.close()
# Sort the output MSs so that they can be compared
myms = mstool()
myms.open(msfile)
myms.sort('ms_sorted.ms', [
'OBSERVATION_ID', 'ARRAY_ID', 'SCAN_NUMBER', 'FIELD_ID',
'DATA_DESC_ID', 'ANTENNA1', 'ANTENNA2', 'TIME'
])
myms.done()
myms.open(self.mmsfile)
myms.sort('mms_sorted.ms', [
'OBSERVATION_ID', 'ARRAY_ID', 'SCAN_NUMBER', 'FIELD_ID',
'DATA_DESC_ID', 'ANTENNA1', 'ANTENNA2', 'TIME'
])
myms.done()
# Ignore WEIGHT_SPECTRUM and SIGMA_SPECTRUM, which are empty columns
self.assertTrue(
th.compTables('ms_sorted.ms', 'mms_sorted.ms', [
'FLAG', 'FLAG_CATEGORY', 'TIME_CENTROID', 'WEIGHT_SPECTRUM',
'SIGMA_SPECTRUM', 'DATA'
]))
# Compare the DATA column
self.assertTrue(
th.compVarColTables('ms_sorted.ms', 'mms_sorted.ms', 'DATA'))
# The separation axis should be written to the output MMS
sepaxis = ph.axisType(self.mmsfile)
self.assertEqual(sepaxis, 'scan,spw',
'Partition did not write AxisType correctly in MMS')
def test_default(self):
'''Partition: create an MMS with default values in parallel'''
# First split off one scan to run the test faster
split(vis=self.msfile, outputvis='split30.ms', datacolumn='DATA', scan='30')
msfile = 'split30.ms'
partition(vis=msfile, outputvis=self.mmsfile)
self.assertTrue(os.path.exists(self.mmsfile), 'MMS was not created for this test')
# Gather several metadata information
# for the MS
mdlocal1 = msmdtool()
mdlocal1.open(msfile)
ms_rows = mdlocal1.nrows()
ms_nscans = mdlocal1.nscans()
ms_nspws = mdlocal1.nspw()
ms_scans = mdlocal1.scannumbers()
mdlocal1.close()
# for the MMS
mdlocal2 = msmdtool()
mdlocal2.open(self.mmsfile)
mms_rows = mdlocal2.nrows()
mms_nscans = mdlocal2.nscans()
mms_nspws = mdlocal2.nspw()
mms_scans = mdlocal2.scannumbers()
mdlocal2.close()
# Compare the number of rows
self.assertEqual(ms_rows, mms_rows, 'Compare total number of rows in MS and MMS')
self.assertEqual(ms_nscans, mms_nscans, 'Compare number of scans')
self.assertEqual(ms_nspws, mms_nspws, 'Compare number of spws')
# Compare the scans
self.assertEqual(ms_scans.all(), mms_scans.all(), 'Compare all scan IDs')
try:
mdlocal1.open(msfile)
mdlocal2.open(self.mmsfile)
# Compare the spws
for i in ms_scans:
msi = mdlocal1.spwsforscan(i)
mmsi = mdlocal2.spwsforscan(i)
self.assertEqual(msi.all(), mmsi.all(), 'Compare spw Ids for a scan')
finally:
mdlocal1.close()
mdlocal2.close()
# Sort the output MSs so that they can be compared
myms = mstool()
myms.open(msfile)
myms.sort('ms_sorted.ms',['OBSERVATION_ID','ARRAY_ID','SCAN_NUMBER','FIELD_ID','DATA_DESC_ID','ANTENNA1','ANTENNA2','TIME'])
myms.done()
myms.open(self.mmsfile)
myms.sort('mms_sorted.ms',['OBSERVATION_ID','ARRAY_ID','SCAN_NUMBER','FIELD_ID','DATA_DESC_ID','ANTENNA1','ANTENNA2','TIME'])
myms.done()
# Ignore WEIGHT_SPECTRUM and SIGMA_SPECTRUM, which are empty columns
self.assertTrue(th.compTables('ms_sorted.ms', 'mms_sorted.ms',
['FLAG','FLAG_CATEGORY','TIME_CENTROID',
'WEIGHT_SPECTRUM','SIGMA_SPECTRUM','DATA']))
# Compare the DATA column
self.assertTrue(th.compVarColTables('ms_sorted.ms', 'mms_sorted.ms','DATA'))
# The separation axis should be written to the output MMS
sepaxis = ph.axisType(self.mmsfile)
self.assertEqual(sepaxis, 'scan,spw', 'Partition did not write AxisType correctly in MMS')
def test_default(self):
"""Partition: create an MMS with default values in parallel"""
# First split off one scan to run the test faster
split(vis=self.msfile, outputvis="split30.ms", datacolumn="DATA", scan="30")
msfile = "split30.ms"
partition(vis=msfile, outputvis=self.mmsfile)
self.assertTrue(os.path.exists(self.mmsfile), "MMS was not created for this test")
# Gather several metadata information
# for the MS
mdlocal1 = msmdtool()
mdlocal1.open(msfile)
ms_rows = mdlocal1.nrows()
ms_nscans = mdlocal1.nscans()
ms_nspws = mdlocal1.nspw()
ms_scans = mdlocal1.scannumbers()
mdlocal1.close()
# for the MMS
mdlocal2 = msmdtool()
mdlocal2.open(self.mmsfile)
mms_rows = mdlocal2.nrows()
mms_nscans = mdlocal2.nscans()
mms_nspws = mdlocal2.nspw()
mms_scans = mdlocal2.scannumbers()
mdlocal2.close()
# Compare the number of rows
self.assertEqual(ms_rows, mms_rows, "Compare total number of rows in MS and MMS")
self.assertEqual(ms_nscans, mms_nscans, "Compare number of scans")
self.assertEqual(ms_nspws, mms_nspws, "Compare number of spws")
# Compare the scans
self.assertEqual(ms_scans.all(), mms_scans.all(), "Compare all scan IDs")
try:
mdlocal1.open(msfile)
mdlocal2.open(self.mmsfile)
# Compare the spws
for i in ms_scans:
msi = mdlocal1.spwsforscan(i)
mmsi = mdlocal2.spwsforscan(i)
self.assertEqual(msi.all(), mmsi.all(), "Compare spw Ids for a scan")
finally:
mdlocal1.close()
mdlocal2.close()
# Sort the output MSs so that they can be compared
myms = mstool()
myms.open(msfile)
myms.sort(
"ms_sorted.ms",
["OBSERVATION_ID", "ARRAY_ID", "SCAN_NUMBER", "FIELD_ID", "DATA_DESC_ID", "ANTENNA1", "ANTENNA2", "TIME"],
)
myms.done()
myms.open(self.mmsfile)
myms.sort(
"mms_sorted.ms",
["OBSERVATION_ID", "ARRAY_ID", "SCAN_NUMBER", "FIELD_ID", "DATA_DESC_ID", "ANTENNA1", "ANTENNA2", "TIME"],
)
myms.done()
self.assertTrue(
th.compTables(
"ms_sorted.ms", "mms_sorted.ms", ["FLAG", "FLAG_CATEGORY", "TIME_CENTROID", "WEIGHT_SPECTRUM", "DATA"]
)
)
# Compare the DATA column
self.assertTrue(th.compVarColTables("ms_sorted.ms", "mms_sorted.ms", "DATA"))
# The separation axis should be written to the output MMS
sepaxis = ph.axisType(self.mmsfile)
self.assertEqual(sepaxis, "scan,spw", "Partition did not write AxisType correctly in MMS")
def plot_weight_density(vis,
spw=0,
field='',
nbins=50,
bins=None,
clear=False,
ignore_flags=False,
representative_channel=None,
**kwargs):
"""
Plot the "weight density" vs uvdist: i.e., the sum of the weights in each
annular bin divided by the area of that bin
Parameters
----------
vis : str
The .ms table to plot weights from
spw : int or str
The spectral window to plot. Only one spectral window should be specified.
field : str
The field name to plot (if mosaic, make sure it is a name and not a number)
nbins : int
The number of bins to create
bins : None or array
You can specify specific bins to average the weights in
ignore_flags : bool
Ignore the flags in the file. Flagged data will be plotted alongside
unflagged.
representative_channel : None or int
A specific channel from which to extract flags. If left as 'None',
defaults to the mean frequency
kwargs : dict
Keyword arguments are passed to msselect (e.g., obsid). Unfortunately,
it seems that msselect will happily ignore just about everything it is
given.
"""
if hasattr(spw, '__len__'):
assert len(spw) == 0, "Only one SPW can be plotted."
mymsmd = msmdtool()
mymsmd.open(vis)
reffreq = "{value}{unit}".format(**mymsmd.reffreq(spw)['m0'])
reffreq = "{0}Hz".format(mymsmd.meanfreq(spw))
if representative_channel is not None:
closest_channel = representative_channel
else:
closest_channel = np.argmin(
np.abs(mymsmd.chanfreqs(spw) - mymsmd.meanfreq(spw)))
mymsmd.close()
myms = mstool()
myms.open(vis)
myms.selectinit(0)
selection_dict = dict(field=field) #, spw=reffreq,)
selection_dict.update(kwargs)
#print(selection_dict)
assert myms.msselect(selection_dict), "Data selection has failed"
#print(myms.msselectedindices())
# select one "representative" channel out of the SPW (because the weights
# are per SPW, but the flags are per channel)
assert myms.selectchannel(start=closest_channel, nchan=1, inc=1,
width=1), "Channel selection has failed"
if ignore_flags:
columns = ['UVW', 'WEIGHT']
else:
columns = ['UVW', 'WEIGHT', 'FLAG']
datadict = myms.getdata(columns)
myms.close()
wt = datadict['weight'].squeeze()
uvw = datadict['uvw'].squeeze()
# calculate the UV distance from the uvw array
uvd = (uvw[:2, :]**2).sum(axis=0)**0.5
if bins is None:
bins = np.linspace(uvd.min(), uvd.max(), nbins)
if not ignore_flags:
# We have exactly one channel (we forced it above) and the second index
# should be the channel ID
# If the flag shape does not conform to this assumption, we're in trouble
# squeeze just gets rid of all size=1 dimensions
flags = datadict['flag'].squeeze()
if flags.shape != wt.shape:
raise ValueError("Flag shape and weight shape don't match. "
"Flag shape: {0} Weight shape: {1}".format(
flags.shape, wt.shape))
# set weights to zero because we're adding them (this is obviously not right
# for many operations, but it is right here!)
wt[flags] = 0
# one plot for each polarization
h_1 = np.histogram(uvd, bins, weights=wt[0, :])
h_2 = np.histogram(uvd, bins, weights=wt[1, :])
# plot points at the bin center
midbins = (bins[:-1] + bins[1:]) / 2.
# compute the bin area for division below
bin_area = (bins[1:]**2 - bins[:-1]**2) * np.pi
if clear:
pl.clf()
pl.plot(midbins, h_1[0] / bin_area, drawstyle='steps-mid')
pl.plot(midbins, h_2[0] / bin_area, drawstyle='steps-mid')
pl.xlabel("UV Distance")
pl.ylabel("Sum of weights / annular area")
def tsysTransfer(vis,
scaleSpws='',
tsysTable='',
newTsysTable='',
verbose=False,
overwrite=True,
printAntenna=0,
printPol=0):
"""
Generate a new Tsys table where the entries for one field are propagated to
other fields which do not have a measured Tsys, using autocorr
(linear!) or SQLD data to determine the change in Tsys.
Input:
vis the MS
scaleSpws the autocorr or SQLD SpWs to use for scaling (integer list or
comma-delimited string, default is the channel-averaged science spws)
tsysTable: if blank, then try vis+'.tsys'
newTsysTable: if blank, then try vis+'.newtsys'
printAntenna: print the before/after values for this antenna ID
printPol: print the before/after values for this polarization (0 or 1)
Returns: nothing
"""
# intents likely to imply different attenuations or tuning to science-like
# scans that we are applying Tsys to.
badIntents = [
'CALIBRATE_POINTING', 'CALIBRATE_FOCUS', 'CALIBRATE_SIDEBAND_RATIO',
'CALIBRATE_ATMOSPHERE'
]
if type(scaleSpws) == str:
if (len(scaleSpws) > 0):
scaleSpws = [int(i) for i in scaleSpws.split(',')]
if (tsysTable == ''):
tsysTable = vis + '.tsys'
if not os.path.exists(tsysTable):
tsysTables = glob.glob(os.path.join(vis, '*tsyscal.tbl'))
if len(tsysTables) < 1:
print("Could not find any tsys tables.")
return
tsysTable = tsysTables[0]
if not os.path.exists(tsysTable):
print("Could not find tsys table: %s" % (tsysTable))
return
if (newTsysTable == ''):
newTsysTable = vis + '.newtsys'
if overwrite and os.path.exists(newTsysTable):
print("Removing pre-existing newTsysTable: ", newTsysTable)
rmtables(newTsysTable)
if os.path.exists(newTsysTable):
shutil.rmtree(newTsysTable)
if (not os.path.exists(tsysTable)):
print("Cannot find Tsys table: ", tsysTable)
return
if (not os.path.exists(vis)):
print("Cannot find measurement set: ", vis)
return
t = time.time()
mytb = taskinit.tbtool()
mymsmd = taskinit.msmdtool()
mytb.open(tsysTable, nomodify=False)
mymsmd.open(vis)
print("tsysTransfer: initial setup took %.3f seconds" % (time.time() - t))
# For convenience squish the useful columns into unique lists
t = time.time()
tsysSpws = pb.unique(mytb.getcol("SPECTRAL_WINDOW_ID"))
tsysBasebands = getBasebands(mymsmd, tsysSpws)
tsysScans = pb.unique(mytb.getcol("SCAN_NUMBER"))
tsysTimes = pb.unique(mytb.getcol("TIME"))
tsysFields = pb.unique(mytb.getcol("FIELD_ID"))
tsysAntennas = pb.unique(mytb.getcol("ANTENNA1"))
finalScan = np.max(mymsmd.scannumbers())
print("Tsys SpWs (%d):" % len(tsysSpws), tsysSpws)
print("Tsys Basebands (%d):" % len(tsysSpws), tsysBasebands)
print("Tsys Scans (%d):" % len(tsysScans), tsysScans)
print("Tsys Times (%d):" % len(tsysTimes), tsysTimes)
print("Tsys Fields (%d):" % len(tsysFields), tsysFields)
print("Tsys Antennas (%d):" % len(tsysAntennas), tsysAntennas)
if (len(scaleSpws) == 0):
# number of scaleSpws should not exceed number of Tsys spws
scaleSpws = np.unique(getChannelAveragedScienceSpws(vis,
mymsmd=mymsmd))
scaleBasebands = getBasebands(mymsmd, scaleSpws)
if scaleBasebands != tsysBasebands:
print("re-ordering scaleSpws to match Tsys basebands")
newScaleSpws = []
for baseband in tsysBasebands:
newScaleSpws.append(scaleSpws[scaleBasebands.index(baseband)])
scaleSpws = newScaleSpws
scaleBasebands = tsysBasebands[:]
print("Getting power from spws: ", scaleSpws)
tsysScanTimes = {}
for s in tsysScans:
st = mytb.query('SCAN_NUMBER==%d' % s)
ts = st.getcol("TIME")
st.close()
tsysScanTimes[s] = sum(ts) / float(len(ts))
if verbose:
print("Tsys scan %d assumed time: %.4f" % (s, tsysScanTimes[s]))
refPowers = {}
refScans = {}
tsysScansOnField = {}
for f in tsysFields:
scanFieldsTab = mytb.query('FIELD_ID==%d' % f)
fieldTsysScans = pb.unique(scanFieldsTab.getcol("SCAN_NUMBER"))
scanFieldsTab.close()
tsysScansOnField[f] = fieldTsysScans
fieldAllScans = mymsmd.scansforfield(f)
fieldName = mymsmd.namesforfields(f)[0]
fieldNonTsysScans = [
x for x in fieldAllScans if x not in fieldTsysScans
]
if (len(fieldNonTsysScans) < 1):
# Then there is no non-tsys scan for this field, e.g. which can happen in a mosaic where the Tsys scan has a different field ID,
# but in this case the field name will have other scans with different field IDs, so revert to using field names. Using field
# names might work from the outset, but I have not tried it.
fieldAllScans = mymsmd.scansforfield(fieldName)
fieldNonTsysScans = [
x for x in fieldAllScans if x not in fieldTsysScans
]
if (len(fieldNonTsysScans) < 1):
print(
"****** This field (id=%d, name=%s) appears to have no non-Tsys-like-scans, and thus cannot be normalized."
% (f, fieldName))
return -1
print("Field %d (%s) Tsys scans:" % (f, fieldName), fieldTsysScans,
", All scans:", fieldAllScans, ", Non-Tsys scans:",
fieldNonTsysScans)
scienceLikeScans = []
for s in fieldNonTsysScans:
intents = mymsmd.intentsforscan(s)
good = True
for i in intents:
for b in badIntents:
if i.startswith(b):
good = False
break
if good: scienceLikeScans.append(s)
powerRefScans = []
for s in fieldTsysScans:
minDist = 9999999
refScan = -1
for r in scienceLikeScans:
dist = abs(r - s)
if dist < minDist:
minDist = dist
refScan = r
powerRefScans.append(refScan)
if verbose:
print("Field %d (%s) Tsys scans:" % (f, fieldName), fieldTsysScans,
", All scans:", fieldAllScans, ", Non-Tsys scans:",
fieldNonTsysScans, ", Non-Tsys science-like scans:",
scienceLikeScans)
for i in range(len(fieldTsysScans)):
if verbose:
print(" Tsys scan %3d power reference scan: %3d" %
(fieldTsysScans[i], powerRefScans[i]))
refScans[fieldTsysScans[i]] = powerRefScans[i]
if verbose:
print(
"populating powers corresponding to each Tsys scan on field %d..."
% (f))
for i in range(len(fieldTsysScans)):
refPowers[fieldTsysScans[i]] = []
for spw in scaleSpws:
if verbose:
print("powerRefScans: ", powerRefScans)
print("calling getPower(vis, %d, %d, 10.0, %s)" %
(powerRefScans[i], spw,
str(powerRefScans[i] < fieldTsysScans[i])))
p = getPower(vis,
powerRefScans[i],
spw,
10.0,
powerRefScans[i] < fieldTsysScans[i],
verbose=verbose)
refPowers[fieldTsysScans[i]].append(p)
#print "powers to use for Tsys scan %d:"%fieldTsysScans[i], refPowers[fieldTsysScans[i]]
# print refPowers
print("tsysTransfer: summarising Tsys table took %.3f seconds" %
(time.time() - t))
t = time.time()
mytb.copy(newTsysTable)
mytb.close()
# re-open original table as read-only
mytb.open(tsysTable)
mytbNew = taskinit.tbtool()
mytbNew.open(newTsysTable, nomodify=False)
print(
"tsysTransfer: Copying Tsys table from '%s' to '%s' took %.3f seconds"
% (tsysTable, newTsysTable, time.time() - t))
anyProcessingNeeded = False
# Loop over each Tsys scan
for i in range(len(tsysScans) - 1):
tsysTime0 = tsysScanTimes[tsysScans[i]]
tsysTime1 = tsysScanTimes[tsysScans[i + 1]]
tsysTimeGap = tsysTime1 - tsysTime0
tsysFields0 = mymsmd.fieldsforscan(tsysScans[i]) # current Tsys scan
tsysFields1 = mymsmd.fieldsforscan(tsysScans[i + 1]) # next Tsys scan
# loop over all scans between the current Tsys scan and the next one
startScan = tsysScans[i] + 1
stopScan = tsysScans[i + 1]
# if finalScan > stopScan and i==len(tsysScans)-1:
# print "There are more scans after the final Tsys scan, extending the range of scans accordingly."
# stopScan = finalScan
for scan in range(startScan, stopScan):
if 'CALIBRATE_POINTING#ON_SOURCE' in mymsmd.intentsforscan(scan):
continue
processingNeeded = False
fields = mymsmd.fieldsforscan(scan)
times = mymsmd.timesforscan(scan)
startTime = times[0]
endTime = times[-1]
print(
"Processing scan %d with fields %s, between Tsys scan %d (fields %s) and %d (fields %s)"
% (scan, str(fields[0]), tsysScans[i], str(
tsysFields0[0]), tsysScans[i + 1], str(tsysFields1[0])))
print(
" Scan %d starts %.3f sec after preceding Tsys, and ends %.3f sec before next Tsys"
% (scan, startTime - tsysTime0, tsysTime1 - endTime))
# There are a few possible cases to deal with:
# 1) this was a power reference scan for a Tsys scan, in which case only produce one extra Tsys, at the opposite end of the scan, or none if there are Tsys scans for the same field at both ends
fieldMatchesPriorTsysField = fieldsMatch(fields, tsysFields0)
fieldMatchesNextTsysField = fieldsMatch(fields, tsysFields1)
priorScanIsTsys = scan == tsysScans[i] + 1
nextScanIsTsys = scan == tsysScans[i + 1] - 1
bracketingTsysFieldsMatch = fieldsMatch(tsysFields0, tsysFields1)
scanIsNotRefScan = scan != refScans[
tsysScans[i]] and scan != refScans[tsysScans[i + 1]]
if fieldMatchesPriorTsysField and fieldMatchesNextTsysField and priorScanIsTsys and nextScanIsTsys:
print(
" Nothing needed for scan %d as bracketed immediately by two Tsys scans of same field"
% scan)
# The most common case for wanting to do the transfer: science field bracketed by phase cal, or phase cal without Tsys immediately before/after
elif bracketingTsysFieldsMatch and (not fieldMatchesPriorTsysField
or scanIsNotRefScan):
# The two Tsys scans that bracket this scan are taken on the same field;
# and either this scan is not on the field of the prior Tsys scan, or
# this scan is not a reference scan
processingNeeded = True
priorScanToUse = tsysScans[i]
nextScanToUse = tsysScans[i + 1]
elif (not bracketingTsysFieldsMatch
and fields[0] in tsysScansOnField.keys()):
candidateScans = np.array(tsysScansOnField[fields[0]])
if (scan < candidateScans[0] or scan > candidateScans[-1]):
print(
" The bracketing Tsys fields do not match, and there are not two scans to interpolate between."
)
else:
processingNeeded = True
priorScanToUse = np.max(
candidateScans[np.where(candidateScans < scan)])
nextScanToUse = np.min(
candidateScans[np.where(candidateScans > scan)])
print(
" The bracketing Tsys fields do not match, but there are two scans to interpolate between: %d and %d."
% (priorScanToUse, nextScanToUse))
elif (not bracketingTsysFieldsMatch):
# This section added by Todd for initial phase calibrator scans when Tsys taken on science target only.
# Not sure what to do yet, though.
print(
" The bracketing Tsys fields do not match, and Tsys was never taken on this field. No processing will be done."
)
if False:
processingNeeded = True
if i + 1 < len(tsysScans):
print(
" Extrapolating from subsequent Tsys scan: %d" %
(tsysScans[i + 1]))
priorScanToUse = tsysScans[i + 1]
nextScanToUse = tsysScans[i + 1]
else:
print(" Extrapolating from prior Tsys scan: %d" %
(tsysScans[i + 1]))
priorScanToUse = tsysScans[i]
nextScanToUse = tsysScans[i]
else:
print(
" This scan arrangement is unexpected. No processing will be done."
)
print(" bracketingTsysFieldsMatch = %s" %
bracketingTsysFieldsMatch)
print(" fieldMatchesPriorTsysField = %s" %
fieldMatchesPriorTsysField)
print(" fieldMatchesNextTsysField = %s" %
fieldMatchesNextTsysField)
print(" priorScanIsTsys = %s" % priorScanIsTsys)
print(" nextScanIsTsys = %s" % nextScanIsTsys)
print(" scanIsNotRefScan = %s" % scanIsNotRefScan)
print(" %s in tsysScansOnField(%s) = %s" %
(fields[0], tsysScansOnField.keys(), fields[0]
in tsysScansOnField.keys()))
if processingNeeded:
anyProcessingNeeded = True
print(
" For scan %d will generate two Tsys entries for beginning and end of scan, interpolating reference from scans %d and %d"
% (scan, priorScanToUse, nextScanToUse))
for ispw in range(len(scaleSpws)):
spw = scaleSpws[ispw]
startPower = getPower(vis,
scan,
spw,
10.0,
False,
verbose=verbose)
endPower = getPower(vis,
scan,
spw,
10.0,
True,
verbose=verbose)
for ant in range(len(tsysAntennas)):
tsysSubTab0 = mytb.query(
"SCAN_NUMBER==%d AND SPECTRAL_WINDOW_ID==%d AND ANTENNA1==%d"
% (priorScanToUse, tsysSpws[ispw], ant))
tsysSubTab1 = mytb.query(
"SCAN_NUMBER==%d AND SPECTRAL_WINDOW_ID==%d AND ANTENNA1==%d"
% (nextScanToUse, tsysSpws[ispw], ant))
# sanity check for duplicate entries
if tsysSubTab0.nrows() != 1 or tsysSubTab1.nrows(
) != 1:
print(
"WARNING!!! not one result row for (scan,ant,spw) query in Tsys table. Scan %d: %d rows, Scan %d: %d rows."
% (priorScanToUse, tsysSubTab0.nrows(),
nextScanToUse, tsysSubTab1.nrows()))
tsys0 = tsysSubTab0.getcell('FPARAM', 0)
tsys1 = tsysSubTab1.getcell('FPARAM', 0)
tsysSubTab1.close()
startTsys = copy.copy(tsys0)
endTsys = copy.copy(
tsys0
) # just a placeholder, new values will be filled in below
startRefPower = refPowers[priorScanToUse]
endRefPower = refPowers[nextScanToUse]
tsysTime0 = tsysScanTimes[priorScanToUse]
tsysTime1 = tsysScanTimes[nextScanToUse]
tsysTimeGap = tsysTime1 - tsysTime0
for pol in range(len(tsys0)):
for chan in range(len(tsys0[pol])):
startTsys0 = TsysAfterPowerChange(
startRefPower[ispw][ant][pol],
startPower[ant][0], tsys0[pol][chan])
startTsys1 = TsysAfterPowerChange(
endRefPower[ispw][ant][pol],
startPower[ant][0], tsys1[pol][chan])
endTsys0 = TsysAfterPowerChange(
startRefPower[ispw][ant][pol],
endPower[ant][0], tsys0[pol][chan])
endTsys1 = TsysAfterPowerChange(
endRefPower[ispw][ant][pol],
endPower[ant][0], tsys1[pol][chan])
if tsysTimeGap == 0:
startTsys[pol][chan] = startTsys0
endTsys[pol][chan] = endTsys0
else:
startTsys[pol][chan] = (
(startTime - tsysTime0) * startTsys1 +
(tsysTime1 - startTime) *
startTsys0) / tsysTimeGap
endTsys[pol][chan] = (
(endTime - tsysTime0) * endTsys1 +
(tsysTime1 - endTime) *
endTsys0) / tsysTimeGap
if chan == len(
tsys0[pol]
) / 2 and ant == printAntenna and pol == printPol:
print(
" ispw=%d spw=%d ant=%d pol=%d chan=%d: TsysBefore: %.1f K, TsysScanStart: %.1f K (interp %.1f,%.1f), TsysScanEnd: %.1f K (interp %.1f,%.1f), TsysAfter: %.1f K"
%
(ispw, spw, ant, pol, chan,
tsys0[pol][chan],
startTsys[pol][chan], startTsys0,
startTsys1, endTsys[pol][chan],
endTsys0, endTsys1, tsys1[pol][chan]))
for f in fields:
nr = mytbNew.nrows()
tsysSubTab0.copyrows(newTsysTable, nrow=1)
if verbose:
print("setting tsys at row %d" % nr)
mytbNew.putcell('FPARAM', nr, startTsys)
mytbNew.putcell('TIME', nr, startTime)
mytbNew.putcell('FIELD_ID', nr, f)
mytbNew.putcell('SCAN_NUMBER', nr, scan)
nr = mytbNew.nrows()
tsysSubTab0.copyrows(newTsysTable, nrow=1)
if verbose:
print("setting tsys at row %d" % nr)
mytbNew.putcell('FPARAM', nr, endTsys)
mytbNew.putcell('TIME', nr, endTime)
mytbNew.putcell('FIELD_ID', nr, f)
mytbNew.putcell('SCAN_NUMBER', nr, scan)
tsysSubTab0.close()
# end loop over fields (f)
# end loop over antennas (ant)
# end loop over spws (ispw)
# end if processingNeeded
# end loop over scans between tsysScans (scan)
mytbNew.flush()
# end loop over Tsys scans
if not anyProcessingNeeded:
print(
"Because no processing was needed the new Tsys table is identical to the original."
)
# TODO: These cleanups should be done also on an exception too
print("Closing tables...")
mytbNew.unlock()
mytbNew.close()
mytbNew.done()
mymsmd.close()
mytb.close()
mytb.done()
def tsysNormalize(vis,
tsysTable='',
newTsysTable='',
scaleSpws=[],
verbose=False):
"""
Generate Tsys entries for one field from other fields, using autocorr
(linear!) or SQLD data to determine the change in Tsys.
Inputs:
vis the MS
tsysTable: the tsys caltable (default = <vis>.tsys)
newTsysTable: the new tsys caltable to create (default = <tsysTable>_normalized)
"""
# intents likely to imply different attenuations or tuning to science-like
# scans that we are applying Tsys to.
print("Entered")
badIntents = [
'CALIBRATE_POINTING', 'CALIBRATE_FOCUS', 'CALIBRATE_SIDEBAND_RATIO',
'CALIBRATE_ATMOSPHERE'
]
if (tsysTable == ''):
tsysTable = vis + '.tsys'
if (not os.path.exists(tsysTable)):
print("Cannot find Tsys table: ", tsysTable)
return
if (not os.path.exists(vis)):
print("Cannot find measurement set: ", vis)
return
t = time.time()
mytb = taskinit.tbtool()
mymsmd = taskinit.msmdtool()
mytb.open(tsysTable, nomodify=False)
mymsmd.open(vis)
print("tsysNormalize: initial setup took %.3f seconds" % (time.time() - t))
# For convenience squish the useful columns into unique lists
t = time.time()
tsysSpws = pb.unique(mytb.getcol("SPECTRAL_WINDOW_ID"))
tsysScans = pb.unique(mytb.getcol("SCAN_NUMBER"))
tsysTimes = pb.unique(mytb.getcol("TIME"))
tsysFields = pb.unique(mytb.getcol("FIELD_ID"))
tsysAntennas = pb.unique(mytb.getcol("ANTENNA1"))
if type(scaleSpws) == str:
scaleSpws = [int(i) for i in scaleSpws.split(',')]
if len(scaleSpws) < len(tsysSpws):
scaleSpws = []
for tsysSpw in tsysSpws:
scaleSpws.append(scienceSpwForTsysSpw(mymsmd, tsysSpw))
print("Identified autocorrelation spws to use: ", scaleSpws)
print("Tsys Spws (%d):" % len(tsysSpws), tsysSpws)
print("Tsys Scans (%d):" % len(tsysScans), tsysScans)
print("Tsys Times (%d):" % len(tsysTimes), tsysTimes)
print("Tsys Fields (%d):" % len(tsysFields), tsysFields)
print("Tsys Antennas (%d):" % len(tsysAntennas), tsysAntennas)
# Gather the power levels to use in the normalization process
refPowers = {}
refScans = {}
for f in tsysFields:
scanFieldsTab = mytb.query('FIELD_ID==%d' % f)
fieldTsysScans = pb.unique(scanFieldsTab.getcol("SCAN_NUMBER"))
scanFieldsTab.close()
fieldAllScans = mymsmd.scansforfield(f)
fieldNonTsysScans = [
x for x in fieldAllScans if x not in fieldTsysScans
]
fieldName = mymsmd.namesforfields(f)[0]
if (len(fieldNonTsysScans) < 1):
# Then there is no non-tsys scan for this field, e.g. which can happen in a mosaic where the Tsys scan has a different field ID,
# but in this case the field name will have other scans with different field IDs, so revert to using field names. Using field
# names might work from the outset, but I have not tried it.
fieldAllScans = mymsmd.scansforfield(fieldName)
fieldNonTsysScans = [
x for x in fieldAllScans if x not in fieldTsysScans
]
if (len(fieldNonTsysScans) < 1):
print(
"****** This field (id=%d, name=%s) appears to have no non-Tsys-like-scans, and thus cannot be normalized."
% (f, fieldName))
return -1
scienceLikeScans = []
for s in fieldNonTsysScans:
intents = mymsmd.intentsforscan(s)
good = True
for i in intents:
for b in badIntents:
if i.startswith(b):
good = False
break
if good: scienceLikeScans.append(s)
powerRefScans = []
for s in fieldTsysScans:
minDist = 9999999
refScan = -1
for r in scienceLikeScans:
dist = abs(r - s)
if dist < minDist:
minDist = dist
refScan = r
powerRefScans.append(refScan)
print("Field %d (%s) Tsys scans:" % (f, fieldname), fieldTsysScans,
", All scans:", fieldAllScans, ", Non-Tsys scans:",
fieldNonTsysScans, ", Non-Tsys science-like scans:",
scienceLikeScans)
for i in range(len(fieldTsysScans)):
print(" Tsys scan %3d power reference scan: %3d" %
(fieldTsysScans[i], powerRefScans[i]))
refScans[fieldTsysScans[i]] = powerRefScans[i]
if verbose:
print(
"populating powers corresponding to each Tsys scan on field %d..."
% (f))
for i in range(len(fieldTsysScans)):
refPowers[fieldTsysScans[i]] = []
for spw in scaleSpws:
if verbose:
print("calling getPower(vis, %d, %d, 10.0, %s)" %
(powerRefScans[i], spw,
str(powerRefScans[i] < fieldTsysScans[i])))
p = getPower(vis,
powerRefScans[i],
spw,
10.0,
powerRefScans[i] < fieldTsysScans[i],
verbose=verbose)
refPowers[fieldTsysScans[i]].append(p)
if verbose:
print("powers to use for Tsys scan %d:" % fieldTsysScans[i],
refPowers[fieldTsysScans[i]])
if verbose: print(refPowers)
print("tsysNormalize: summarising Tsys table took %.3f seconds" %
(time.time() - t))
t = time.time()
# Now copy the original Tsys caltable and update all the values in the new one.
if (newTsysTable == ''):
newTsysTable = tsysTable + '_normalized'
if (os.path.exists(newTsysTable)):
shutil.rmtree(newTsysTable)
mytb.copy(newTsysTable)
mytb.close()
mytb.open(newTsysTable, nomodify=False)
startRefPower = refPowers[tsysScans[0]]
for i in range(1, len(tsysScans)):
# need to adjust each successive Tsys
refPower = refPowers[tsysScans[i]]
for ispw in range(len(tsysSpws)):
spw = tsysSpws[ispw]
for ant in range(len(tsysAntennas)):
tsysSubTab1 = mytb.query(
"SCAN_NUMBER==%d AND SPECTRAL_WINDOW_ID==%d AND ANTENNA1==%d"
% (tsysScans[i], tsysSpws[ispw], ant))
tsys1 = tsysSubTab1.getcell('FPARAM', 0)
newTsys = tsysSubTab1.getcell('FPARAM', 0)
for pol in range(len(tsys1)):
for chan in range(len(tsys1[pol])):
a = TsysAfterPowerChange(
refPowers[tsysScans[i]][ispw][ant][pol],
startRefPower[ispw][ant][pol], tsys1[pol][chan])
newTsys[pol][chan] = a
print("Scan %2d spw %2d pol %d mean %.1f --> %.1f" %
(tsysScans[i], spw, pol, np.mean(
tsys1[pol]), np.mean(newTsys[pol])))
tsysSubTab1.putcell('FPARAM', 0, newTsys)
tsysSubTab1.close()
mymsmd.close()
mytb.close()
def plot_weight_density(vis, spw=0, field='', nbins=50, bins=None, clear=False,
ignore_flags=False, representative_channel=None,
**kwargs):
"""
Plot the "weight density" vs uvdist: i.e., the sum of the weights in each
annular bin divided by the area of that bin
Parameters
----------
vis : str
The .ms table to plot weights from
spw : int or str
The spectral window to plot. Only one spectral window should be specified.
field : str
The field name to plot (if mosaic, make sure it is a name and not a number)
nbins : int
The number of bins to create
bins : None or array
You can specify specific bins to average the weights in
ignore_flags : bool
Ignore the flags in the file. Flagged data will be plotted alongside
unflagged.
representative_channel : None or int
A specific channel from which to extract flags. If left as 'None',
defaults to the mean frequency
kwargs : dict
Keyword arguments are passed to msselect (e.g., obsid). Unfortunately,
it seems that msselect will happily ignore just about everything it is
given.
"""
if hasattr(spw, '__len__'):
assert len(spw) == 0, "Only one SPW can be plotted."
mymsmd = msmdtool()
mymsmd.open(vis)
reffreq = "{value}{unit}".format(**mymsmd.reffreq(spw)['m0'])
reffreq = "{0}Hz".format(mymsmd.meanfreq(spw))
if representative_channel is not None:
closest_channel = representative_channel
else:
closest_channel = np.argmin(np.abs(mymsmd.chanfreqs(spw) - mymsmd.meanfreq(spw)))
mymsmd.close()
myms = mstool()
myms.open(vis)
myms.selectinit(0)
selection_dict = dict(field=field) #, spw=reffreq,)
selection_dict.update(kwargs)
#print(selection_dict)
assert myms.msselect(selection_dict), "Data selection has failed"
#print(myms.msselectedindices())
# select one "representative" channel out of the SPW (because the weights
# are per SPW, but the flags are per channel)
assert myms.selectchannel(start=closest_channel, nchan=1, inc=1, width=1), "Channel selection has failed"
if ignore_flags:
columns = ['UVW', 'WEIGHT']
else:
columns = ['UVW', 'WEIGHT', 'FLAG']
datadict=myms.getdata(columns)
myms.close()
wt = datadict['weight'].squeeze()
uvw = datadict['uvw'].squeeze()
# calculate the UV distance from the uvw array
uvd = (uvw[:2,:]**2).sum(axis=0)**0.5
if bins is None:
bins = np.linspace(uvd.min(), uvd.max(), nbins)
if not ignore_flags:
# We have exactly one channel (we forced it above) and the second index
# should be the channel ID
# If the flag shape does not conform to this assumption, we're in trouble
# squeeze just gets rid of all size=1 dimensions
flags = datadict['flag'].squeeze()
if flags.shape != wt.shape:
raise ValueError("Flag shape and weight shape don't match. "
"Flag shape: {0} Weight shape: {1}".format(
flags.shape,wt.shape))
# set weights to zero because we're adding them (this is obviously not right
# for many operations, but it is right here!)
wt[flags] = 0
# one plot for each polarization
h_1 = np.histogram(uvd, bins, weights=wt[0,:])
h_2 = np.histogram(uvd, bins, weights=wt[1,:])
# plot points at the bin center
midbins = (bins[:-1] + bins[1:])/2.
# compute the bin area for division below
bin_area = (bins[1:]**2-bins[:-1]**2)*np.pi
if clear:
pl.clf()
pl.plot(midbins, h_1[0]/bin_area, drawstyle='steps-mid')
pl.plot(midbins, h_2[0]/bin_area, drawstyle='steps-mid')
pl.xlabel("UV Distance")
pl.ylabel("Sum of weights / annular area")
def subvs2(vis=None,
outputvis=None,
timerange='',
spw='',
mode=None,
subtime1=None,
subtime2=None,
smoothaxis=None,
smoothtype=None,
smoothwidth=None,
splitsel=None,
reverse=None,
overwrite=None):
"""Perform vector subtraction for visibilities
Keyword arguments:
vis -- Name of input visibility file (MS)
default: none; example: vis='ngc5921.ms'
outputvis -- Name of output uv-subtracted visibility file (MS)
default: none; example: outputvis='ngc5921_src.ms'
timerange -- Time range of performing the UV subtraction:
default='' means all times. examples:
timerange = 'YYYY/MM/DD/hh:mm:ss~YYYY/MM/DD/hh:mm:ss'
timerange = 'hh:mm:ss~hh:mm:ss'
spw -- Select spectral window/channel.
default = '' all the spectral channels. Example: spw='0:1~20'
mode -- operation mode
default 'linear'
mode = 'linear': use a linear fit for the background to be subtracted
mode = 'lowpass': act as a lowpass filter---smooth the data using different
smooth types and window sizes. Can be performed along either time
or frequency axis
mode = 'highpass': act as a highpass filter---smooth the data first, and
subtract the smoothed data from the original. Can be performed along
either time or frequency axis
mode = 'linear' expandable parameters:
subtime1 -- Time range 1 of the background to be subtracted from the data
default='' means all times. format:
timerange = 'YYYY/MM/DD/hh:mm:ss~YYYY/MM/DD/hh:mm:ss'
timerange = 'hh:mm:ss~hh:mm:ss'
subtime2 -- Time range 2 of the backgroud to be subtracted from the data
default='' means all times. examples:
timerange = 'YYYY/MM/DD/hh:mm:ss~YYYY/MM/DD/hh:mm:ss'
timerange = 'hh:mm:ss~hh:mm:ss'
mode = 'lowpass' or 'highpass' expandable parameters:
smoothaxis -- axis of smooth
Default: 'time'
smoothaxis = 'time': smooth is along the time axis
smoothaxis = 'freq': smooth is along the frequency axis
smoothtype -- type of the smooth depending on the convolving kernel
Default: 'flat'
smoothtype = 'flat': convolving kernel is a flat rectangle,
equivalent to a boxcar moving smooth
smoothtype = 'hanning': Hanning smooth kernel. See numpy.hanning
smoothtype = 'hamming': Hamming smooth kernel. See numpy.hamming
smoothtype = 'bartlett': Bartlett smooth kernel. See numpy.bartlett
smoothtype = 'blackman': Blackman smooth kernel. See numpy.blackman
smoothwidth -- width of the smooth kernel
Default: 5
Examples: smoothwidth=5, meaning the width is 5 pixels
splitsel -- True or False. default = False. If splitsel = False, then the entire input
measurement set is copied as the output measurement set (outputvis), with
background subtracted at selected timerange and spectral channels.
If splitsel = True,then only the selected timerange and spectral channels
are copied into the output measurement set (outputvis).
reverse -- True or False. default = False. If reverse = False, then the times indicated
by subtime1 and/or subtime2 are treated as background and subtracted; If reverse
= True, then reverse the sign of the background-subtracted data. The option can
be used for mapping absorptive structure.
overwrite -- True or False. default = False. If overwrite = True and
outputvis already exists, the selected subtime and spw in the
output measurment set will be replaced with background subtracted
visibilities
"""
# check the visbility ms
casalog.post('input parameters:')
casalog.post('vis: ' + vis)
casalog.post('outputvis: ' + outputvis)
casalog.post('smoothaxis: ' + smoothaxis)
casalog.post('smoothtype: ' + smoothtype)
casalog.post('smoothwidth: ' + str(smoothwidth))
if not outputvis or outputvis.isspace():
raise (ValueError, 'Please specify outputvis')
if os.path.exists(outputvis):
if overwrite:
print(
"The already existing output measurement set will be updated.")
else:
raise (ValueError,
"Output MS %s already exists - will not overwrite." %
outputvis)
else:
if not splitsel:
shutil.copytree(vis, outputvis)
else:
ms.open(vis, nomodify=True)
ms.split(outputvis, spw=spw, time=timerange, whichcol='DATA')
ms.close()
if timerange and (type(timerange) == str):
[btimeo, etimeo] = timerange.split('~')
btimeosec = qa.getvalue(qa.convert(qa.totime(btimeo), 's'))
etimeosec = qa.getvalue(qa.convert(qa.totime(etimeo), 's'))
timebinosec = etimeosec - btimeosec
if timebinosec < 0:
raise Exception(
'Negative timebin! Please check the "timerange" parameter.')
casalog.post('Selected timerange: ' + timerange +
' as the time for UV subtraction.')
else:
casalog.post(
'Output timerange not specified, using the entire timerange')
if spw and (type(spw) == str):
spwlist = spw.split(';')
else:
casalog.post('spw not specified, use all frequency channels')
# read the output data
datams = mstool()
datams.open(outputvis, nomodify=False)
datamsmd = msmdtool()
datamsmd.open(outputvis)
spwinfod = datams.getspectralwindowinfo()
spwinfok = spwinfod.keys()
spwinfok.sort(key=int)
spwinfol = [spwinfod[k] for k in spwinfok]
for s, spi in enumerate(spwinfol):
print('processing spectral window {}'.format(spi['SpectralWindowId']))
datams.selectinit(reset=True)
staql = {'time': '', 'spw': ''}
if not splitsel:
# outputvis is identical to input visibility, do the selection
if timerange and (type(timerange == str)):
staql['time'] = timerange
if spw and (type(spw) == str):
staql['spw'] = spwlist[s]
if not spw and not timerange:
# data selection is not made
print('selecting all spws and times')
staql['spw'] = str(spi['SpectralWindowId'])
else:
# outputvis is splitted, selections have already applied, select all the data
print('split the selected spws and times')
staql['spw'] = str(spi['SpectralWindowId'])
datams.msselect(staql)
orec = datams.getdata(['data', 'time', 'axis_info'], ifraxis=True)
npol, nchan, nbl, ntim = orec['data'].shape
print('dimension of output data', orec['data'].shape)
casalog.post('Number of baselines: ' + str(nbl))
casalog.post('Number of spectral channels: ' + str(nchan))
casalog.post('Number of time pixels: ' + str(ntim))
try:
if mode == 'linear':
# define and check the background time ranges
if subtime1 and (type(subtime1) == str):
[bsubtime1, esubtime1] = subtime1.split('~')
bsubtime1sec = qa.getvalue(
qa.convert(qa.totime(bsubtime1), 's'))
esubtime1sec = qa.getvalue(
qa.convert(qa.totime(esubtime1), 's'))
timebin1sec = esubtime1sec - bsubtime1sec
if timebin1sec < 0:
raise Exception(
'Negative timebin! Please check the "subtime1" parameter.'
)
casalog.post('Selected timerange 1: ' + subtime1 +
' as background for uv subtraction.')
else:
raise Exception(
'Please enter at least one timerange as the background'
)
if subtime2 and (type(subtime2) == str):
[bsubtime2, esubtime2] = subtime2.split('~')
bsubtime2sec = qa.getvalue(
qa.convert(qa.totime(bsubtime2), 's'))
esubtime2sec = qa.getvalue(
qa.convert(qa.totime(esubtime2), 's'))
timebin2sec = esubtime2sec - bsubtime2sec
if timebin2sec < 0:
raise Exception(
'Negative timebin! Please check the "subtime2" parameter.'
)
timebin2 = str(timebin2sec) + 's'
casalog.post('Selected timerange 2: ' + subtime2 +
' as background for uv subtraction.')
# plus 1s is to ensure averaging over the entire timerange
else:
casalog.post(
'Timerange 2 not selected, using only timerange 1 as background'
)
# Select the background indicated by subtime1
ms.open(vis, nomodify=True)
# Select the spw id
# ms.msselect({'time': subtime1})
staql0 = {'time': subtime1, 'spw': ''}
if spw and (type(spw) == str):
staql0['spw'] = spwlist[s]
else:
staql0['spw'] = staql['spw']
ms.msselect(staql0)
rec1 = ms.getdata(['data', 'time', 'axis_info'], ifraxis=True)
# print('shape of the frequency matrix ',rec1['axis_info']['freq_axis']['chan_freq'].shape)
sz1 = rec1['data'].shape
print('dimension of selected background 1', rec1['data'].shape)
# the data shape is (n_pol,n_channel,n_baseline,n_time), no need to reshape
# rec1['data']=rec1['data'].reshape(sz1[0],sz1[1],sz1[2],nspw,sz1[3]/nspw,order='F')
# print('reshaped rec1 ', rec1['data'].shape)
rec1avg = np.average(rec1['data'], axis=3)
casalog.post('Averaging the visibilities in subtime1: ' +
subtime1)
ms.close()
if subtime2 and (type(subtime2) == str):
ms.open(vis, nomodify=True)
# Select the spw id
staql0 = {'time': subtime2, 'spw': ''}
if spw and (type(spw) == str):
staql0['spw'] = spwlist[s]
else:
staql0['spw'] = staql['spw']
ms.msselect(staql0)
rec2 = ms.getdata(['data', 'time', 'axis_info'],
ifraxis=True)
sz2 = rec2['data'].shape
print('dimension of selected background 2',
rec2['data'].shape)
# rec2['data']=rec2['data'].reshape(sz2[0],sz2[1],sz2[2],nspw,sz2[3]/nspw,order='F')
# print('reshaped rec1 ', rec2['data'].shape)
rec2avg = np.average(rec2['data'], axis=3)
ms.close()
casalog.post('Averaged the visibilities in subtime2: ' +
subtime2)
if subtime1 and (not subtime2):
casalog.post(
'Only "subtime1" is defined, subtracting background defined in subtime1: '
+ subtime1)
t1 = (np.amax(rec1['time']) + np.amin(rec1['time'])) / 2.
print('t1: ',
qa.time(qa.quantity(t1, 's'), form='ymd', prec=10))
for i in range(ntim):
orec['data'][:, :, :, i] -= rec1avg
if reverse:
orec['data'][:, :, :,
i] = -orec['data'][:, :, :, i]
if subtime1 and subtime2 and (type(subtime2) == str):
casalog.post(
'Both subtime1 and subtime2 are specified, doing linear interpolation between "subtime1" and "subtime2"'
)
t1 = (np.amax(rec1['time']) + np.amin(rec1['time'])) / 2.
t2 = (np.amax(rec2['time']) + np.amin(rec2['time'])) / 2.
touts = orec['time']
print('t1: ',
qa.time(qa.quantity(t1, 's'), form='ymd', prec=10))
print('t2: ',
qa.time(qa.quantity(t2, 's'), form='ymd', prec=10))
for i in range(ntim):
tout = touts[i]
if tout > np.amax([t1, t2]):
tout = np.amax([t1, t2])
elif tout < np.amin([t1, t2]):
tout = np.amin([t1, t2])
orec['data'][:, :, :, i] -= (rec2avg - rec1avg) * (
tout - t1) / (t2 - t1) + rec1avg
if reverse:
orec['data'][:, :, :,
i] = -orec['data'][:, :, :, i]
elif mode == 'highpass':
if smoothtype != 'flat' and smoothtype != 'hanning' and smoothtype != 'hamming' and smoothtype != 'bartlett' and smoothtype != 'blackman':
raise Exception('Unknown smoothtype ' + str(smoothtype))
if smoothaxis == 'time':
if smoothwidth <= 0 or smoothwidth >= ntim:
raise Exception(
'Specified smooth width is <=0 or >= the total number of '
+ smoothaxis)
else:
for i in range(orec['data'].shape[0]):
for j in range(nchan):
for k in range(nbl):
orec['data'][i, j,
k, :] -= signalsmooth.smooth(
orec['data'][i, j, k, :],
smoothwidth, smoothtype)
if smoothaxis == 'freq':
if smoothwidth <= 0 or smoothwidth >= nchan:
raise Exception(
'Specified smooth width is <=0 or >= the total number of '
+ smoothaxis)
else:
for i in range(orec['data'].shape[0]):
for j in range(nbl):
for k in range(ntim):
orec['data'][i, :, j,
k] -= signalsmooth.smooth(
orec['data'][i, :, j, k],
smoothwidth, smoothtype)
elif mode == 'lowpass':
if smoothtype != 'flat' and smoothtype != 'hanning' and smoothtype != 'hamming' and smoothtype != 'bartlett' and smoothtype != 'blackman':
raise Exception('Unknown smoothtype ' + str(smoothtype))
if smoothaxis == 'time':
if smoothwidth <= 0 or smoothwidth >= ntim:
raise Exception(
'Specified smooth width is <=0 or >= the total number of '
+ smoothaxis)
else:
for i in range(orec['data'].shape[0]):
for j in range(nchan):
for k in range(nbl):
orec['data'][i, j,
k, :] = signalsmooth.smooth(
orec['data'][i, j, k, :],
smoothwidth, smoothtype)
if smoothaxis == 'freq':
if smoothwidth <= 0 or smoothwidth >= nchan:
raise Exception(
'Specified smooth width is <=0 or >= the total number of '
+ smoothaxis)
else:
for i in range(orec['data'].shape[0]):
for j in range(nbl):
for k in range(ntim):
orec['data'][i, :, j,
k] = signalsmooth.smooth(
orec['data'][i, :, j, k],
smoothwidth, smoothtype)
else:
raise Exception('Unknown mode' + str(mode))
except Exception as instance:
print('*** Error ***', instance)
# orec['data']=orec['data'].reshape(szo[0],szo[1],szo[2],szo[3],order='F')
# put the modified data back into the output visibility set
del orec['time']
del orec['axis_info']
# ms.open(outputvis,nomodify=False)
# if not splitsel:
# outputvis is identical to input visibility, do the selection
# if timerange and (type(timerange==str)):
# datams.msselect({'time':timerange})
# if spw and (type(spw)==str):
# datams.selectinit(datadescid=int(spwid))
# nchan=int(echan)-int(bchan)+1
# datams.selectchannel(nchan,int(bchan),1,1)
# if not spw and not timerange:
# data selection is not made
# datams.selectinit(datadescid=0)
# else:
# outputvis is splitted, selections have already applied, select all the data
# datams.selectinit(datadescid=0)
datams.putdata(orec)
datams.close()
datamsmd.done()