Esempio n. 1
0
    def test_mms_XXYY_selection(self):
        '''mstransform: correlation='RR,LL' should select and re-index properly'''
        self.outputms = '3cRRLL.mms'
        # spw 0 should not be processed. The selection should happen before the MMS work
        mstransform(vis=self.vis, outputvis=self.outputms, datacolumn='data', correlation='RR,LL',
                    createmms=True, separationaxis='auto')
        
        msmdt = msmdtool()
        msmdt.open(self.outputms)
        out_dds = msmdt.datadescids()
        msmdt.done()
        
        ref = [0,1]
        for i in out_dds:
            self.assertEqual(out_dds[i], ref[i])
        
        pol_col = th.getVarCol(self.outputms+'/POLARIZATION','NUM_CORR')
        self.assertEqual(pol_col['r1'][0], 0,'Error in NUM_CORR of POLARIZATION table')
        self.assertEqual(pol_col['r2'][0], 0,'Error in NUM_CORR of POLARIZATION table')
        self.assertEqual(pol_col['r3'][0], 2,'Error in NUM_CORR of POLARIZATION table')
        self.assertEqual(pol_col['r4'][0], 2,'Error in NUM_CORR of POLARIZATION table')

        # Verify that POLARIZATION table is not re-sized.
        corr_col = th.getVarCol(self.outputms+'/POLARIZATION', 'NUM_CORR')
        self.assertEqual(corr_col.keys().__len__(), 4, 'Wrong number of rows in POLARIZATION table')

        # Check the FEED table
#        out_feed_spw = th.getVarCol(self.outputms+'/FEED', 'SPECTRAL_WINDOW_ID')
#        self.assertEqual(len(out_feed_spw.keys()), 52)
        
        # listobs, listpartition should not fail
        listobs(self.outputms, listfile='3c_1.obs')
        self.assertTrue(os.path.exists('3c_1.obs'), 'Probable error in sub-table re-indexing')
        listpartition(self.outputms, listfile='3c_2.obs')
        self.assertTrue(os.path.exists('3c_2.obs'), 'Probable error in sub-table re-indexing')
Esempio n. 2
0
    def test_split_MMS(self):
        '''mstransform: Split MMS in parallel'''
        # Create an MMS in the setup. It creates self.testmms
        self.createMMS(self.vis, axis='scan', spws='0,1')
        
        self.outputms = 'scan30.mms'
        mstransform(vis=self.testmms, outputvis=self.outputms, datacolumn='data', scan='30')
        
        self.assertTrue(ParallelTaskHelper.isParallelMS(self.outputms),'Output is not an MMS')
        
        mslocal = mstool()
        mslocal.open(self.outputms)
        sublist = mslocal.getreferencedtables()
        self.assertEqual(len(sublist), 1)
        
        # Test DD table
        msmdt = msmdtool()
        msmdt.open(self.outputms)
        out_dds = msmdt.datadescids()
        msmdt.done()
        
        ref = [0,1]
        for i in out_dds:
            self.assertEqual(out_dds[i], ref[i])

        # The separation axis should be copied to the output MMS
        in_sepaxis = ph.axisType(self.testmms)
        out_sepaxis = ph.axisType(self.outputms)
        self.assertEqual(in_sepaxis, out_sepaxis, 'AxisTypes from input and output MMS do not match')
Esempio n. 3
0
    def test_mms_scan_spw_partition(self):
        '''mstransform: Create MMS and part by scan/spw'''
        self.outputms = '3cscanspw02.mms'
        mstransform(vis=self.vis, outputvis=self.outputms, datacolumn='data', spw='0,2',
                    createmms=True, disableparallel=True, separationaxis='auto')

        # Verify the input versus the output
        msmdt = msmdtool()
        msmdt.open(self.outputms)
        out_dds = msmdt.datadescids()
        msmdt.done()

        ref = [0,1,2]
        for i in out_dds:
            self.assertEqual(out_dds[i], ref[i])

        # Verify that DATA_DESCRIPTION table is properly re-indexed.
        spw_col = th.getVarCol(self.outputms+'/DATA_DESCRIPTION', 'SPECTRAL_WINDOW_ID')
        self.assertEqual(spw_col.keys().__len__(), 3, 'Wrong number of rows in DD table')
        self.assertEqual(spw_col['r1'][0], 0,'Error re-indexing SPECTRAL_WINDOW_ID of DATA_DESCRIPTION table')
        self.assertEqual(spw_col['r2'][0], 0,'Error re-indexing SPECTRAL_WINDOW_ID of DATA_DESCRIPTION table')
        self.assertEqual(spw_col['r3'][0], 1,'Error re-indexing SPECTRAL_WINDOW_ID of DATA_DESCRIPTION table')
        
        # Check the FEED table
        out_feed_spw = th.getVarCol(self.outputms+'/FEED', 'SPECTRAL_WINDOW_ID')
        self.assertEqual(out_feed_spw['r1'],[0])
        self.assertEqual(out_feed_spw['r26'],[0])
        self.assertEqual(out_feed_spw['r27'],[1])
        self.assertEqual(out_feed_spw['r28'],[1])
        self.assertEqual(out_feed_spw['r51'],[1])
        self.assertEqual(len(out_feed_spw.keys()), 52)
Esempio n. 4
0
    def test_split_MMS_weight_corr_sel(self):
        '''mstransform: Split MMS in parallel. Check WEIGHT shape when selecting correlation'''
        # Create an MMS in the setup. It creates self.testmms
        self.createMMS(self.vis, axis='scan', spws='0,1')
        
        self.outputms = 'corrRR_LL.mms'
        mstransform(vis=self.testmms, outputvis=self.outputms, datacolumn='data', correlation='RR,LL',spw='0')
        
        self.assertTrue(ParallelTaskHelper.isParallelMS(self.outputms),'Output is not an MMS')
        
        mslocal = mstool()
        mslocal.open(self.outputms)
        sublist = mslocal.getreferencedtables()
        self.assertEqual(len(sublist), 2)
        
        # Test DD table
        msmdt = msmdtool()
        msmdt.open(self.outputms)
        out_dds = msmdt.datadescids()
        msmdt.done()
        
        ref = [0]
        for i in out_dds:
            self.assertEqual(out_dds[i], ref[i])

        # The separation axis should be copied to the output MMS
        in_sepaxis = ph.axisType(self.testmms)
        out_sepaxis = ph.axisType(self.outputms)
        self.assertEqual(in_sepaxis, out_sepaxis, 'AxisTypes from input and output MMS do not match')

        # Check the dimensions of the WEIGHT and SIGMA columns. CAS-6946
        out_ws = th.getColShape(self.outputms,'WEIGHT')
        out_ss = th.getColShape(self.outputms,'SIGMA')
        self.assertEqual(out_ws[0],'[2]','WEIGHT shape is not correct')
        self.assertEqual(out_ss[0],'[2]','SIGMA shape is not correct')
Esempio n. 5
0
def getPlotantsObservatoryInfo(msname):
	metadata = msmdtool()
	metadata.open(msname)
	telescope = metadata.observatorynames()[0]
	arrayPos = metadata.observatoryposition()
	metadata.close()
	return telescope, arrayPos
Esempio n. 6
0
    def test_float_data_mms_baseline_auto(self):
        '''importasdm: Create an MMS with a FLOAT_DATA column separated per baseline '''
        myasdmname = 'uid___A002_X6218fb_X264'
        themsname = myasdmname + ".ms"

        # The ocorr_mode='ao' option will create a FLOAT_DATA column instead of DATA
        importasdm(myasdmname,
                   vis=themsname,
                   ocorr_mode='ao',
                   createmms=True,
                   scans='1',
                   separationaxis='baseline',
                   numsubms=4)
        self.assertTrue(ParallelDataHelper.isParallelMS(themsname),
                        'Output is not a Multi-MS')
        self.assertTrue(len(th.getColDesc(themsname, 'FLOAT_DATA')) > 0)

        md = msmdtool()
        md.open(themsname)
        bsl = md.baselines()
        md.close()

        #diagnoals give the auto-corrs
        ac = bsl.diagonal()
        self.assertTrue(ac.all(), 'Not all auto-correlations are there')

        # Take the dictionary and compare with original MS
        thisdict = listpartition(vis=themsname, createdict=True)
        self.assertEqual(len(thisdict.keys()), 4,
                         'There should be 4 subMSs in output MMS')
Esempio n. 7
0
    def test_mms_spw_selection3(self):
        '''mstransform: Create MMS and select three spws with numsubms=2'''
        self.outputms = '3cspw012.mms'
        mstransform(vis=self.vis, outputvis=self.outputms, datacolumn='data', spw='0,1,2',
                    createmms=True, separationaxis='spw', numsubms=2)

        # Verify the input versus the output
        msmdt = msmdtool()
        msmdt.open(self.outputms)
        out_dds = msmdt.datadescids()
        out_nrow = msmdt.nrows()
        msmdt.done()

        self.assertTrue(out_nrow,5200)
        ref = [0,1,2,3]
        for i in out_dds:
            self.assertEqual(out_dds[i], ref[i])

        # Verify that DATA_DESCRIPTION table is properly re-indexed.
        spw_col = th.getVarCol(self.outputms+'/DATA_DESCRIPTION', 'SPECTRAL_WINDOW_ID')
        self.assertEqual(spw_col.keys().__len__(), 4, 'Wrong number of rows in DD table')
        self.assertEqual(spw_col['r1'][0], 0,'Error re-indexing SPECTRAL_WINDOW_ID of DATA_DESCRIPTION table')
        self.assertEqual(spw_col['r2'][0], 0,'Error re-indexing SPECTRAL_WINDOW_ID of DATA_DESCRIPTION table')
        self.assertEqual(spw_col['r3'][0], 1,'Error re-indexing SPECTRAL_WINDOW_ID of DATA_DESCRIPTION table')
        self.assertEqual(spw_col['r4'][0], 2,'Error re-indexing SPECTRAL_WINDOW_ID of DATA_DESCRIPTION table')
        
        in_feed_tb = th.getVarCol(self.vis+'/FEED', 'SPECTRAL_WINDOW_ID')
        out_feed_tb = th.getVarCol(self.outputms+'/FEED', 'SPECTRAL_WINDOW_ID')
        
        # Check the FEED table
        th.compTables(self.vis+'/FEED', self.outputms+'/FEED', ['FOCUS_LENGTH'])
Esempio n. 8
0
    def freq_selection_overlap(ms, freqsel, spw=0):
        """
        For a given frequency selection string (e.g., '215~216GHz;900~950GHz'),
        find the subset of the `;`-separated entries that overlap with the
        measurement set and return those.

        Parameters
        ----------
        ms : str
            The name of a measurement set to compare against
        freqsel : str
            A frequency-based selection string
        spw : int
            The spectral window number, default to 0
        """

        # use the default (topo or lsrk or whatever) frequency, don't cvel it,
        # because the inputs are assumed to be in the same frame.
        # it's important to be exact here, since we're using the fmin/fmax
        # to fill in when one of the ends of a selection is out of range.
        msmd = msmdtool()
        msmd.open(ms)
        frequencies_in_ms = msmd.chanfreqs(spw)
        msmd.close()

        fmin, fmax = frequencies_in_ms.min(), frequencies_in_ms.max()

        new_sel = []

        for selstr in freqsel.split(";"):
            lo, hi = map(float, selstr.strip(string.ascii_letters).split("~"))
            unit = selstr.lstrip(string.punctuation + string.digits)
            if lo > hi:
                lo, hi = hi, lo

            flo = qq.convert({'value': float(lo), 'unit': unit}, 'Hz')['value']
            fhi = qq.convert({'value': float(hi), 'unit': unit}, 'Hz')['value']

            if ((fhi < fmax) and (fhi > fmin)) and ((flo > fmin) and
                                                    (flo < fmax)):
                # if the whole thing is in range...
                new_sel.append(selstr)
            elif ((fhi < fmax) and (fhi > fmin)) and not ((flo > fmin) and
                                                          (flo < fmax)):
                # if the lower end is out of range, but the upper is in
                new_selstr = "{0}~{1}GHz".format(fmin / 1e9, fhi / 1e9)
                new_sel.append(new_selstr)
            elif (not ((fhi < fmax) and (fhi > fmin))) and ((flo > fmin) and
                                                            (flo < fmax)):
                # if the upper end is out of range, but the lower is in
                new_selstr = "{0}~{1}GHz".format(flo / 1e9, fmax / 1e9)
                new_sel.append(new_selstr)

        return "{0}:".format(spw) + ";".join(new_sel)
Esempio n. 9
0
 def test_baseline_autocorr(self):
     '''partition: create an MMS per baseline axis only for auto-correlations'''
     self.outputms = 'baseline_autocorr.mms'
     partition(self.vis, outputvis=self.outputms, separationaxis='baseline', antenna='*&&&', flagbackup=False)
     
     md = msmdtool()
     md.open(self.outputms)
     bsl = md.baselines()
     md.close()
     
     #diagnoals give the auto-corrs
     ac = bsl.diagonal()
     self.assertTrue(ac.all(), 'Not all auto-correlations are there')
Esempio n. 10
0
 def test_combspws_timespan_spw_axis_error(self):
     '''mstransform: combinespws=True, timespan=scan axis=spw'''
     self.createMMS(self.vis, axis='spw',scans='30',spws='10,11')
     self.outputms = "spwaxiserror.mms"
     # subMSs do not have all spws. Create an MS
     mstransform(vis=self.testmms, outputvis=self.outputms, datacolumn='data',
                 combinespws=True, timeaverage=True, timebin='20s',timespan='scan')
     self.assertFalse(ParallelDataHelper.isParallelMS(self.outputms),'Output should be an MS')
     mymsmd = msmdtool()
     mymsmd.open(self.outputms)
     nspw = mymsmd.nspw()
     mymsmd.close()
     self.assertEqual(nspw,1)
Esempio n. 11
0
 def test_combspws_timespan_error(self):
     '''mstransform: combinespws=True, timespan=scan axis=auto timebin=40s'''
     self.createMMS(self.vis, axis='auto',spws='1,3', numms=4)
     self.outputms = "spanscan_comb.mms"
     # combinespws is not possible. It should create and MS
     mstransform(vis=self.testmms, outputvis=self.outputms, datacolumn='data',
                 combinespws=True, timeaverage=True, timebin='40s',timespan='scan')
     self.assertFalse(ParallelDataHelper.isParallelMS(self.outputms),'Output should be an MS')
     mymsmd = msmdtool()
     mymsmd.open(self.outputms)
     nspw = mymsmd.nspw()
     mymsmd.close()
     self.assertEqual(nspw,1)
Esempio n. 12
0
 def test_combspws_timespan_fail(self):
     '''mstransform: combinespws=True, timespan=scan axis=auto timebin=200s'''
     self.createMMS(self.vis, axis='auto',spws='3')
     self.outputms = "errormms.mms"
     # Scans are shorter than timebin. Create an MS
     mstransform(vis=self.testmms, outputvis=self.outputms, datacolumn='data',
                 combinespws=True, timeaverage=True, timebin='200s',timespan='scan')
     self.assertFalse(ParallelDataHelper.isParallelMS(self.outputms),'Output should be an MS')
     mymsmd = msmdtool()
     mymsmd.open(self.outputms)
     nscan = mymsmd.nscans()
     exposure = mymsmd.exposuretime(30)['value']
     mymsmd.close()
     self.assertEqual(nscan,1)
     self.assertEqual(exposure, 179)
Esempio n. 13
0
 def test_timespan_spw_axis(self):
     '''mstransform: timeaverage=True, timespan=scan, separationaxis=spw'''
     self.createMMS(self.vis, axis='spw',spws='1,3')
     self.outputms = "spanscan_spw.mms"
     mstransform(vis=self.testmms, outputvis=self.outputms, datacolumn='data',timeaverage=True, 
                 timebin='100s',timespan='scan')
     self.assertTrue(ParallelDataHelper.isParallelMS(self.outputms),'Output should be an MMS')
     self.assertEqual(ph.axisType(self.outputms),'spw')
     
     mymsmd = msmdtool()
     mymsmd.open(self.outputms)
     t30 = mymsmd.exposuretime(30)['value']
     t31 = mymsmd.exposuretime(31)['value']
     mymsmd.close()
     self.assertEqual(t30, 100)
     self.assertEqual(t31, 79)
Esempio n. 14
0
def freq_selection_overlap(ms, freqsel, spw=0):
    """
    For a given frequency selection string (e.g., '215~216GHz;900~950GHz'),
    find the subset of the `;`-separated entries that overlap with the
    measurement set and return those.

    Parameters
    ----------
    ms : str
        The name of a measurement set to compare against
    freqsel : str
        A frequency-based selection string
    spw : int
        The spectral window number, default to 0
    """

    msmd = msmdtool()
    msmd.open(ms)
    frequencies_in_ms = msmd.chanfreqs(spw)
    msmd.close()

    fmin, fmax = frequencies_in_ms.min(), frequencies_in_ms.max()

    new_sel = []

    for selstr in freqsel.split(";"):
        lo, hi = map(float, selstr.strip(string.ascii_letters).split("~"))
        unit = selstr.lstrip(string.punctuation + string.digits)
        if lo > hi:
            lo, hi = hi, lo

        flo = qq.convert({'value': float(lo), 'unit': unit}, 'Hz')['value']
        fhi = qq.convert({'value': float(hi), 'unit': unit}, 'Hz')['value']

        if ((fhi < fmax) and (fhi > fmin)) and ((flo > fmin) and (flo < fmax)):
            new_sel.append(selstr)
        elif ((fhi < fmax) and (fhi > fmin)) and not ((flo > fmin) and
                                                      (flo < fmax)):
            new_selstr = "{0}~{1}GHz".format(fmin / 1e9, fhi / 1e9)
            new_sel.append(new_selstr)
        elif (not ((fhi < fmax) and (fhi > fmin))) and ((flo > fmin) and
                                                        (flo < fmax)):
            new_selstr = "{0}~{1}GHz".format(flo / 1e9, fmax / 1e9)
            new_sel.append(new_selstr)

    return "{0}:".format(spw) + ";".join(new_sel)
Esempio n. 15
0
    def test_sd_data_mms_baseline_all(self):
        '''importasdm: Create an MMS separated per baseline, using default numsubms '''
        myasdmname = 'uid___A002_X6218fb_X264'
        themsname = myasdmname+".ms"

        # Create a single-dish MMS with auto and cross correlations
        importasdm(myasdmname, vis=themsname, ocorr_mode='ca', createmms=True, scans='1', separationaxis='baseline')
        self.assertTrue(ParallelDataHelper.isParallelMS(themsname), 'Output is not a Multi-MS')        
        self.assertTrue(len(th.getColDesc(themsname, 'DATA')) > 0)
        
        md = msmdtool()
        md.open(themsname)
        bsl = md.baselines()
        md.close()      
        
        # Check if all baselines are in there
        self.assertTrue(bsl.all(), 'Not all baselines are in the MMS')
Esempio n. 16
0
    def test_baseline_autocorr(self):
        '''partition: create an MMS per baseline axis only for auto-correlations'''
        self.outputms = 'baseline_autocorr.mms'
        partition(self.vis,
                  outputvis=self.outputms,
                  separationaxis='baseline',
                  antenna='*&&&',
                  flagbackup=False)

        md = msmdtool()
        md.open(self.outputms)
        bsl = md.baselines()
        md.close()

        #diagnoals give the auto-corrs
        ac = bsl.diagonal()
        self.assertTrue(ac.all(), 'Not all auto-correlations are there')
Esempio n. 17
0
 def test_baseline_partition(self):
     '''partition: create an MMS per baseline for an interferometry MS'''
     partition(vis=self.msfile, outputvis=self.mmsfile, spw='0,1',createmms=True,
               separationaxis='baseline', flagbackup=False, datacolumn='data')  
     
     listpartition(self.mmsfile) 
     
     # This MS doesn't have auto-correlations. It has 4 antennas and 6 baselines (cross)
     # There will be MSSelectionNullSelection SEVERE messages in the logfile because
     # it will try to create SubMSs from auto-correlations. This is not an error!
     md = msmdtool()
     md.open(self.mmsfile)
     bsl = md.baselines()
     md.close()
     
     # diagonal give the auto-corrs
     ac = bsl.diagonal()
     self.assertFalse(ac.all(), 'There should be no auto-correlations in MMS')
Esempio n. 18
0
    def test_float_data_mms_baseline_auto(self):
        '''importasdm: Create an MMS with a FLOAT_DATA column separated per baseline '''
        myasdmname = 'uid___A002_X6218fb_X264'
        themsname = myasdmname+".ms"

        # The ocorr_mode='ao' option will create a FLOAT_DATA column instead of DATA
        importasdm(myasdmname, vis=themsname, ocorr_mode='ao', createmms=True, scans='1', separationaxis='baseline', numsubms=4)
        self.assertTrue(ParallelDataHelper.isParallelMS(themsname), 'Output is not a Multi-MS')        
        self.assertTrue(len(th.getColDesc(themsname, 'FLOAT_DATA')) > 0)
        
        md = msmdtool()
        md.open(themsname)
        bsl = md.baselines()
        md.close()
        
        #diagnoals give the auto-corrs
        ac = bsl.diagonal()
        self.assertTrue(ac.all(), 'Not all auto-correlations are there')
        
        # Take the dictionary and compare with original MS
        thisdict = listpartition(vis=themsname, createdict=True)
        self.assertEqual(len(thisdict.keys()), 4, 'There should be 4 subMSs in output MMS')
Esempio n. 19
0
    def test_sd_data_mms_baseline_all(self):
        '''importasdm: Create an MMS separated per baseline, using default numsubms '''
        myasdmname = 'uid___A002_X6218fb_X264'
        themsname = myasdmname + ".ms"

        # Create a single-dish MMS with auto and cross correlations
        importasdm(myasdmname,
                   vis=themsname,
                   ocorr_mode='ca',
                   createmms=True,
                   scans='1',
                   separationaxis='baseline')
        self.assertTrue(ParallelDataHelper.isParallelMS(themsname),
                        'Output is not a Multi-MS')
        self.assertTrue(len(th.getColDesc(themsname, 'DATA')) > 0)

        md = msmdtool()
        md.open(themsname)
        bsl = md.baselines()
        md.close()

        # Check if all baselines are in there
        self.assertTrue(bsl.all(), 'Not all baselines are in the MMS')
Esempio n. 20
0
    def test_baseline_partition(self):
        '''partition: create an MMS per baseline for an interferometry MS'''
        partition(vis=self.msfile,
                  outputvis=self.mmsfile,
                  spw='0,1',
                  createmms=True,
                  separationaxis='baseline',
                  flagbackup=False,
                  datacolumn='data')

        listpartition(self.mmsfile)

        # This MS doesn't have auto-correlations. It has 4 antennas and 6 baselines (cross)
        # There will be MSSelectionNullSelection SEVERE messages in the logfile because
        # it will try to create SubMSs from auto-correlations. This is not an error!
        md = msmdtool()
        md.open(self.mmsfile)
        bsl = md.baselines()
        md.close()

        # diagonal give the auto-corrs
        ac = bsl.diagonal()
        self.assertFalse(ac.all(),
                         'There should be no auto-correlations in MMS')
Esempio n. 21
0
if sys.version_info.major > 2:
    from casatools import ms, quanta, msmetadata
    from casatasks import casalog

    casalog.showconsole(True)
    datams = ms()
    ms_in = ms()
    datamsmd = msmetadata()
    qa = quanta()
else:
    from taskinit import ms, qa, mstool, msmdtool, casalog

    datams = mstool()
    ms_in = mstool()
    datamsmd = msmdtool()

# from taskinit import *
# from callibrary import *
# import pdb


def subvs2(vis=None,
           outputvis=None,
           timerange='',
           spw='',
           mode='linear',
           subtime1='',
           subtime2='',
           smoothaxis='time',
           smoothtype='flat',
Esempio n. 22
0
if os.getenv('ALMAIMF_ROOTDIR') is None:
    try:
        import metadata_tools
        os.environ['ALMAIMF_ROOTDIR'] = os.path.split(
            metadata_tools.__file__)[0]
    except ImportError:
        raise ValueError("metadata_tools not found on path; make sure to "
                         "specify ALMAIMF_ROOTDIR environment variable "
                         "or your PYTHONPATH variable to include the directory"
                         " containing the ALMAIMF code.")
else:
    sys.path.append(os.getenv('ALMAIMF_ROOTDIR'))

from metadata_tools import check_channel_flags

msmd = msmdtool()
ms = mstool()
tb = tbtool()

# band name : frequency range (GHz)
bands = {
    'B3': (80, 110),
    'B6': (210, 250),
}


def logprint(string):
    casalog.post(string, origin='split_line_windows')
    print(string)

Esempio n. 23
0
def genImageName(vis='', spw='', field='', imtype='mfs', targettype='sci', stokes='I', mous='', modtext='manual'):
    # DPetry, Sept 2017

    """
    Generate image name to be used in clean command
    following the ALMA archive conventions (see SCIREQ-110, draft recommendations v4.8).
      vis         = MS to be imaged (to extract object name, SPW frequencies
      spw         = ID of SPW to be imaged
                    If several SPWs are imaged at once, enter list of SPWs.
      field       = ID of field to be imaged (to extract object name).
                    in case of mosaic, give any ID of field belonging to the mosaic
      imtype      = image type, 'mfs' (default) - aggregate bandwidth
                    'cont' - mfs with lines excluded
                    'cube' - spectrally resolved cube
      targettype  = 'sci' (default) for science target
                    'bp' for bandpass calibrator
                    'ph' for phase calibrator
                    'chk' for checksource
                    'amp' for flux calibrator
      stokes      = 'I' (default) for Stokes I image
                    any combination of 'I', 'Q', 'U', 'V', 'P', 'A'
      mous        = '' (default) normally this is filled by the archive.
                    But if you know the MOUS UID, enter it in the format with "_"
                    e.g.  uid___A001_X888_X66
      modtext     = 'manual' (default) - this string can be used to add text,
                    i.e. a modification, to the standard file name.
                    It is appended at the end of the name.
    """

    themous = ''
    theobject = ''
    thetargettype = ''
    targettypes = ['sci', 'bp', 'ph', 'chk', 'amp']
    thespwid = ''
    theimtype = ''
    imtypes = ['mfs', 'cont', 'cube']
    thestokes = ''
    stokess = ['I', 'Q', 'U', 'V', 'IQU', 'IQUV', 'P', 'A']
    themodtext = ''

    rval = ''

    mymsmd = taskinit.msmdtool()

    # MOUS
    if (mous != ''):
        themous = mous+'.'

    # object
    try:
        mymsmd.open(vis)
    except:
        mymsmd.done()
        raise Exception("ERROR: problem opening vis")

    myfieldnames = mymsmd.fieldnames()
    if type(field)==str and (field in myfieldnames):
        theobject = field
    elif field in range(mymsmd.nfields()):
        theobject = myfieldnames[field]
    else:
        print "ERROR: invalid field: "+field
        print "Valid entries are ", myfieldnames
        print " or ", range(mymsmd.nfields())
        raise Exception("ERROR: invalid field: "+field)
    myspws = mymsmd.spwsforfield(theobject)
    mymsmd.close()

    # target type
    if targettype in targettypes:
        thetargettype = targettype
    else:
        print "ERROR: invalid targettype ", targettype
        print "  Valid entries are ", targettypes
        raise  Exception("ERROR: invalid targettype ", targettype)

    # spw
    if type(spw) != type([]):
        spw = [spw]
    for myspw in spw:
        if myspw in myspws:
            if thespwid=='':
                thespwid = str(myspw)
            else:
                thespwid = thespwid+'_'+str(myspw)                
        else:
            if not type(myspw) == int:
                print "ERROR: invalid spw: "+str(myspw)+". Valid entries are ", myspws
                raise Exception("ERROR: invalid spw: "+str(myspw)+' Data type must be int.')
            else:
                print "ERROR: invalid spw: "+str(myspw)+". Valid entries are ", myspws
                raise Exception("ERROR: invalid spw: "+str(myspw))

    # imtype
    if imtype in imtypes:
        theimtype = imtype
    else:
        print "ERROR: invalid imtype ", imtype
        print "  Valid entries are ", imtypes
        raise Exception( "ERROR: invalid imtype "+str(imtype))

    # stokes
    if stokes in stokess:
        thestokes = stokes
    else:
        print "ERROR: invalid stokes ", stokes
        print "  Valid entries are ", stokess
        raise Exception("ERROR: invalid stokes "+str(stokes))

    # modifying text

    if modtext != '':
        themodtext = '.'+modtext
        if ' ' in themodtext:
            raise Exception("ERROR: modtext must not contain spaces")

    rval = themous+theobject+'_'+thetargettype+'.spw'+thespwid+'.'+theimtype+'.'+thestokes+themodtext

    if ' ' in rval:
        raise Exception("ERROR: generated name contains spaces: \""+rval+"\"")
    if '/' in rval:
        raise Exception("ERROR: generated name contains slash: \""+rval+"\"")
    if '*' in rval:
        raise Exception("ERROR: generated name contains star: \""+rval+"\"")
    if '?' in rval:
        raise Exception("ERROR: generated name contains question mark: \""+rval+"\"") 

    return rval
Esempio n. 24
0
    def test_default(self):
        '''Partition: create an MMS with default values in parallel'''

        # First split off one scan to run the test faster
        split(vis=self.msfile,
              outputvis='split30.ms',
              datacolumn='DATA',
              scan='30')
        msfile = 'split30.ms'

        partition(vis=msfile, outputvis=self.mmsfile)

        self.assertTrue(os.path.exists(self.mmsfile),
                        'MMS was not created for this test')

        # Gather several metadata information
        # for the MS
        mdlocal1 = msmdtool()
        mdlocal1.open(msfile)
        ms_rows = mdlocal1.nrows()
        ms_nscans = mdlocal1.nscans()
        ms_nspws = mdlocal1.nspw()
        ms_scans = mdlocal1.scannumbers()
        mdlocal1.close()

        # for the MMS
        mdlocal2 = msmdtool()
        mdlocal2.open(self.mmsfile)
        mms_rows = mdlocal2.nrows()
        mms_nscans = mdlocal2.nscans()
        mms_nspws = mdlocal2.nspw()
        mms_scans = mdlocal2.scannumbers()
        mdlocal2.close()

        # Compare the number of rows
        self.assertEqual(ms_rows, mms_rows,
                         'Compare total number of rows in MS and MMS')
        self.assertEqual(ms_nscans, mms_nscans, 'Compare number of scans')
        self.assertEqual(ms_nspws, mms_nspws, 'Compare number of spws')

        # Compare the scans
        self.assertEqual(ms_scans.all(), mms_scans.all(),
                         'Compare all scan IDs')

        try:
            mdlocal1.open(msfile)
            mdlocal2.open(self.mmsfile)

            # Compare the spws
            for i in ms_scans:
                msi = mdlocal1.spwsforscan(i)
                mmsi = mdlocal2.spwsforscan(i)
                self.assertEqual(msi.all(), mmsi.all(),
                                 'Compare spw Ids for a scan')
        finally:
            mdlocal1.close()
            mdlocal2.close()

        # Sort the output MSs so that they can be compared
        myms = mstool()

        myms.open(msfile)
        myms.sort('ms_sorted.ms', [
            'OBSERVATION_ID', 'ARRAY_ID', 'SCAN_NUMBER', 'FIELD_ID',
            'DATA_DESC_ID', 'ANTENNA1', 'ANTENNA2', 'TIME'
        ])
        myms.done()

        myms.open(self.mmsfile)
        myms.sort('mms_sorted.ms', [
            'OBSERVATION_ID', 'ARRAY_ID', 'SCAN_NUMBER', 'FIELD_ID',
            'DATA_DESC_ID', 'ANTENNA1', 'ANTENNA2', 'TIME'
        ])
        myms.done()

        # Ignore WEIGHT_SPECTRUM and SIGMA_SPECTRUM, which are empty columns
        self.assertTrue(
            th.compTables('ms_sorted.ms', 'mms_sorted.ms', [
                'FLAG', 'FLAG_CATEGORY', 'TIME_CENTROID', 'WEIGHT_SPECTRUM',
                'SIGMA_SPECTRUM', 'DATA'
            ]))

        # Compare the DATA column
        self.assertTrue(
            th.compVarColTables('ms_sorted.ms', 'mms_sorted.ms', 'DATA'))

        # The separation axis should be written to the output MMS
        sepaxis = ph.axisType(self.mmsfile)
        self.assertEqual(sepaxis, 'scan,spw',
                         'Partition did not write AxisType correctly in MMS')
Esempio n. 25
0
    def test_default(self):
        '''Partition: create an MMS with default values in parallel'''
        
        # First split off one scan to run the test faster
        split(vis=self.msfile, outputvis='split30.ms', datacolumn='DATA', scan='30')
        msfile = 'split30.ms'

        partition(vis=msfile, outputvis=self.mmsfile)
        
        self.assertTrue(os.path.exists(self.mmsfile), 'MMS was not created for this test')
        
        # Gather several metadata information
        # for the MS
        mdlocal1 = msmdtool()
        mdlocal1.open(msfile)
        ms_rows = mdlocal1.nrows()
        ms_nscans = mdlocal1.nscans()
        ms_nspws = mdlocal1.nspw()
        ms_scans = mdlocal1.scannumbers()
        mdlocal1.close()        
          
        # for the MMS
        mdlocal2 = msmdtool()
        mdlocal2.open(self.mmsfile)
        mms_rows = mdlocal2.nrows()
        mms_nscans = mdlocal2.nscans()
        mms_nspws = mdlocal2.nspw()
        mms_scans = mdlocal2.scannumbers()
        mdlocal2.close()        
          
        # Compare the number of rows
        self.assertEqual(ms_rows, mms_rows, 'Compare total number of rows in MS and MMS')
        self.assertEqual(ms_nscans, mms_nscans, 'Compare number of scans')
        self.assertEqual(ms_nspws, mms_nspws, 'Compare number of spws')
          
        # Compare the scans
        self.assertEqual(ms_scans.all(), mms_scans.all(), 'Compare all scan IDs')
  
        try:
            mdlocal1.open(msfile)
            mdlocal2.open(self.mmsfile)
          
            # Compare the spws
            for i in ms_scans:                
                msi = mdlocal1.spwsforscan(i)
                mmsi = mdlocal2.spwsforscan(i)
                self.assertEqual(msi.all(), mmsi.all(), 'Compare spw Ids for a scan')
        finally:          
            mdlocal1.close()
            mdlocal2.close()               

        # Sort the output MSs so that they can be compared
        myms = mstool()
        
        myms.open(msfile)
        myms.sort('ms_sorted.ms',['OBSERVATION_ID','ARRAY_ID','SCAN_NUMBER','FIELD_ID','DATA_DESC_ID','ANTENNA1','ANTENNA2','TIME'])
        myms.done()
        
        myms.open(self.mmsfile)
        myms.sort('mms_sorted.ms',['OBSERVATION_ID','ARRAY_ID','SCAN_NUMBER','FIELD_ID','DATA_DESC_ID','ANTENNA1','ANTENNA2','TIME'])
        myms.done()

        # Ignore WEIGHT_SPECTRUM and SIGMA_SPECTRUM, which are empty columns
        self.assertTrue(th.compTables('ms_sorted.ms', 'mms_sorted.ms', 
                                      ['FLAG','FLAG_CATEGORY','TIME_CENTROID',
                                       'WEIGHT_SPECTRUM','SIGMA_SPECTRUM','DATA']))

        # Compare the DATA column
        self.assertTrue(th.compVarColTables('ms_sorted.ms', 'mms_sorted.ms','DATA'))
        
        # The separation axis should be written to the output MMS
        sepaxis = ph.axisType(self.mmsfile)
        self.assertEqual(sepaxis, 'scan,spw', 'Partition did not write AxisType correctly in MMS')
Esempio n. 26
0
    def test_default(self):
        """Partition: create an MMS with default values in parallel"""

        # First split off one scan to run the test faster
        split(vis=self.msfile, outputvis="split30.ms", datacolumn="DATA", scan="30")
        msfile = "split30.ms"

        partition(vis=msfile, outputvis=self.mmsfile)

        self.assertTrue(os.path.exists(self.mmsfile), "MMS was not created for this test")

        # Gather several metadata information
        # for the MS
        mdlocal1 = msmdtool()
        mdlocal1.open(msfile)
        ms_rows = mdlocal1.nrows()
        ms_nscans = mdlocal1.nscans()
        ms_nspws = mdlocal1.nspw()
        ms_scans = mdlocal1.scannumbers()
        mdlocal1.close()

        # for the MMS
        mdlocal2 = msmdtool()
        mdlocal2.open(self.mmsfile)
        mms_rows = mdlocal2.nrows()
        mms_nscans = mdlocal2.nscans()
        mms_nspws = mdlocal2.nspw()
        mms_scans = mdlocal2.scannumbers()
        mdlocal2.close()

        # Compare the number of rows
        self.assertEqual(ms_rows, mms_rows, "Compare total number of rows in MS and MMS")
        self.assertEqual(ms_nscans, mms_nscans, "Compare number of scans")
        self.assertEqual(ms_nspws, mms_nspws, "Compare number of spws")

        # Compare the scans
        self.assertEqual(ms_scans.all(), mms_scans.all(), "Compare all scan IDs")

        try:
            mdlocal1.open(msfile)
            mdlocal2.open(self.mmsfile)

            # Compare the spws
            for i in ms_scans:
                msi = mdlocal1.spwsforscan(i)
                mmsi = mdlocal2.spwsforscan(i)
                self.assertEqual(msi.all(), mmsi.all(), "Compare spw Ids for a scan")
        finally:
            mdlocal1.close()
            mdlocal2.close()

        # Sort the output MSs so that they can be compared
        myms = mstool()

        myms.open(msfile)
        myms.sort(
            "ms_sorted.ms",
            ["OBSERVATION_ID", "ARRAY_ID", "SCAN_NUMBER", "FIELD_ID", "DATA_DESC_ID", "ANTENNA1", "ANTENNA2", "TIME"],
        )
        myms.done()

        myms.open(self.mmsfile)
        myms.sort(
            "mms_sorted.ms",
            ["OBSERVATION_ID", "ARRAY_ID", "SCAN_NUMBER", "FIELD_ID", "DATA_DESC_ID", "ANTENNA1", "ANTENNA2", "TIME"],
        )
        myms.done()

        self.assertTrue(
            th.compTables(
                "ms_sorted.ms", "mms_sorted.ms", ["FLAG", "FLAG_CATEGORY", "TIME_CENTROID", "WEIGHT_SPECTRUM", "DATA"]
            )
        )

        # Compare the DATA column
        self.assertTrue(th.compVarColTables("ms_sorted.ms", "mms_sorted.ms", "DATA"))

        # The separation axis should be written to the output MMS
        sepaxis = ph.axisType(self.mmsfile)
        self.assertEqual(sepaxis, "scan,spw", "Partition did not write AxisType correctly in MMS")
Esempio n. 27
0
def plot_weight_density(vis,
                        spw=0,
                        field='',
                        nbins=50,
                        bins=None,
                        clear=False,
                        ignore_flags=False,
                        representative_channel=None,
                        **kwargs):
    """
    Plot the "weight density" vs uvdist: i.e., the sum of the weights in each
    annular bin divided by the area of that bin

    Parameters
    ----------
    vis : str
        The .ms table to plot weights from
    spw : int or str
        The spectral window to plot.  Only one spectral window should be specified.
    field : str
        The field name to plot (if mosaic, make sure it is a name and not a number)
    nbins : int
        The number of bins to create
    bins : None or array
        You can specify specific bins to average the weights in
    ignore_flags : bool
        Ignore the flags in the file.  Flagged data will be plotted alongside
        unflagged.
    representative_channel : None or int
        A specific channel from which to extract flags.  If left as 'None',
        defaults to the mean frequency
    kwargs : dict
        Keyword arguments are passed to msselect (e.g., obsid).  Unfortunately,
        it seems that msselect will happily ignore just about everything it is
        given.
    """

    if hasattr(spw, '__len__'):
        assert len(spw) == 0, "Only one SPW can be plotted."

    mymsmd = msmdtool()
    mymsmd.open(vis)

    reffreq = "{value}{unit}".format(**mymsmd.reffreq(spw)['m0'])
    reffreq = "{0}Hz".format(mymsmd.meanfreq(spw))
    if representative_channel is not None:
        closest_channel = representative_channel
    else:
        closest_channel = np.argmin(
            np.abs(mymsmd.chanfreqs(spw) - mymsmd.meanfreq(spw)))
    mymsmd.close()

    myms = mstool()

    myms.open(vis)
    myms.selectinit(0)
    selection_dict = dict(field=field)  #, spw=reffreq,)
    selection_dict.update(kwargs)
    #print(selection_dict)
    assert myms.msselect(selection_dict), "Data selection has failed"
    #print(myms.msselectedindices())
    # select one "representative" channel out of the SPW (because the weights
    # are per SPW, but the flags are per channel)
    assert myms.selectchannel(start=closest_channel, nchan=1, inc=1,
                              width=1), "Channel selection has failed"
    if ignore_flags:
        columns = ['UVW', 'WEIGHT']
    else:
        columns = ['UVW', 'WEIGHT', 'FLAG']
    datadict = myms.getdata(columns)
    myms.close()
    wt = datadict['weight'].squeeze()
    uvw = datadict['uvw'].squeeze()

    # calculate the UV distance from the uvw array
    uvd = (uvw[:2, :]**2).sum(axis=0)**0.5

    if bins is None:
        bins = np.linspace(uvd.min(), uvd.max(), nbins)

    if not ignore_flags:
        # We have exactly one channel (we forced it above) and the second index
        # should be the channel ID
        # If the flag shape does not conform to this assumption, we're in trouble
        # squeeze just gets rid of all size=1 dimensions
        flags = datadict['flag'].squeeze()

        if flags.shape != wt.shape:
            raise ValueError("Flag shape and weight shape don't match. "
                             "Flag shape: {0}  Weight shape: {1}".format(
                                 flags.shape, wt.shape))

        # set weights to zero because we're adding them (this is obviously not right
        # for many operations, but it is right here!)
        wt[flags] = 0

    # one plot for each polarization
    h_1 = np.histogram(uvd, bins, weights=wt[0, :])
    h_2 = np.histogram(uvd, bins, weights=wt[1, :])

    # plot points at the bin center
    midbins = (bins[:-1] + bins[1:]) / 2.
    # compute the bin area for division below
    bin_area = (bins[1:]**2 - bins[:-1]**2) * np.pi

    if clear:
        pl.clf()
    pl.plot(midbins, h_1[0] / bin_area, drawstyle='steps-mid')
    pl.plot(midbins, h_2[0] / bin_area, drawstyle='steps-mid')
    pl.xlabel("UV Distance")
    pl.ylabel("Sum of weights / annular area")
Esempio n. 28
0
def tsysTransfer(vis,
                 scaleSpws='',
                 tsysTable='',
                 newTsysTable='',
                 verbose=False,
                 overwrite=True,
                 printAntenna=0,
                 printPol=0):
    """
    Generate a new Tsys table where the entries for one field are propagated to
    other fields which do not have a measured Tsys, using autocorr
    (linear!) or SQLD data to determine the change in Tsys.
    Input:
     vis          the MS
     scaleSpws    the autocorr or SQLD SpWs to use for scaling (integer list or
          comma-delimited string, default is the channel-averaged science spws)
     tsysTable:   if blank, then try vis+'.tsys'
     newTsysTable:   if blank, then try vis+'.newtsys'
     printAntenna: print the before/after values for this antenna ID
     printPol: print the before/after values for this polarization (0 or 1)
    Returns: nothing
    """
    # intents likely to imply different attenuations or tuning to science-like
    # scans that we are applying Tsys to.
    badIntents = [
        'CALIBRATE_POINTING', 'CALIBRATE_FOCUS', 'CALIBRATE_SIDEBAND_RATIO',
        'CALIBRATE_ATMOSPHERE'
    ]
    if type(scaleSpws) == str:
        if (len(scaleSpws) > 0):
            scaleSpws = [int(i) for i in scaleSpws.split(',')]
    if (tsysTable == ''):
        tsysTable = vis + '.tsys'
        if not os.path.exists(tsysTable):
            tsysTables = glob.glob(os.path.join(vis, '*tsyscal.tbl'))
            if len(tsysTables) < 1:
                print("Could not find any tsys tables.")
                return
            tsysTable = tsysTables[0]
    if not os.path.exists(tsysTable):
        print("Could not find tsys table: %s" % (tsysTable))
        return
    if (newTsysTable == ''):
        newTsysTable = vis + '.newtsys'
    if overwrite and os.path.exists(newTsysTable):
        print("Removing pre-existing newTsysTable: ", newTsysTable)
        rmtables(newTsysTable)
        if os.path.exists(newTsysTable):
            shutil.rmtree(newTsysTable)
    if (not os.path.exists(tsysTable)):
        print("Cannot find Tsys table: ", tsysTable)
        return
    if (not os.path.exists(vis)):
        print("Cannot find measurement set: ", vis)
        return

    t = time.time()
    mytb = taskinit.tbtool()
    mymsmd = taskinit.msmdtool()
    mytb.open(tsysTable, nomodify=False)
    mymsmd.open(vis)
    print("tsysTransfer: initial setup took %.3f seconds" % (time.time() - t))

    # For convenience squish the useful columns into unique lists
    t = time.time()
    tsysSpws = pb.unique(mytb.getcol("SPECTRAL_WINDOW_ID"))
    tsysBasebands = getBasebands(mymsmd, tsysSpws)
    tsysScans = pb.unique(mytb.getcol("SCAN_NUMBER"))
    tsysTimes = pb.unique(mytb.getcol("TIME"))
    tsysFields = pb.unique(mytb.getcol("FIELD_ID"))
    tsysAntennas = pb.unique(mytb.getcol("ANTENNA1"))
    finalScan = np.max(mymsmd.scannumbers())
    print("Tsys SpWs (%d):" % len(tsysSpws), tsysSpws)
    print("Tsys Basebands (%d):" % len(tsysSpws), tsysBasebands)
    print("Tsys Scans (%d):" % len(tsysScans), tsysScans)
    print("Tsys Times (%d):" % len(tsysTimes), tsysTimes)
    print("Tsys Fields (%d):" % len(tsysFields), tsysFields)
    print("Tsys Antennas (%d):" % len(tsysAntennas), tsysAntennas)
    if (len(scaleSpws) == 0):
        # number of scaleSpws should not exceed number of Tsys spws
        scaleSpws = np.unique(getChannelAveragedScienceSpws(vis,
                                                            mymsmd=mymsmd))
        scaleBasebands = getBasebands(mymsmd, scaleSpws)
        if scaleBasebands != tsysBasebands:
            print("re-ordering scaleSpws to match Tsys basebands")
            newScaleSpws = []
            for baseband in tsysBasebands:
                newScaleSpws.append(scaleSpws[scaleBasebands.index(baseband)])
            scaleSpws = newScaleSpws
            scaleBasebands = tsysBasebands[:]
        print("Getting power from spws: ", scaleSpws)

    tsysScanTimes = {}
    for s in tsysScans:
        st = mytb.query('SCAN_NUMBER==%d' % s)
        ts = st.getcol("TIME")
        st.close()
        tsysScanTimes[s] = sum(ts) / float(len(ts))
        if verbose:
            print("Tsys scan %d assumed time: %.4f" % (s, tsysScanTimes[s]))

    refPowers = {}
    refScans = {}
    tsysScansOnField = {}
    for f in tsysFields:
        scanFieldsTab = mytb.query('FIELD_ID==%d' % f)
        fieldTsysScans = pb.unique(scanFieldsTab.getcol("SCAN_NUMBER"))
        scanFieldsTab.close()
        tsysScansOnField[f] = fieldTsysScans
        fieldAllScans = mymsmd.scansforfield(f)
        fieldName = mymsmd.namesforfields(f)[0]
        fieldNonTsysScans = [
            x for x in fieldAllScans if x not in fieldTsysScans
        ]
        if (len(fieldNonTsysScans) < 1):
            # Then there is no non-tsys scan for this field, e.g. which can happen in a mosaic where the Tsys scan has a different field ID,
            # but in this case the field name will have other scans with different field IDs, so revert to using field names.  Using field
            # names might work from the outset, but I have not tried it.
            fieldAllScans = mymsmd.scansforfield(fieldName)
            fieldNonTsysScans = [
                x for x in fieldAllScans if x not in fieldTsysScans
            ]
            if (len(fieldNonTsysScans) < 1):
                print(
                    "****** This field (id=%d, name=%s) appears to have no non-Tsys-like-scans, and thus cannot be normalized."
                    % (f, fieldName))
                return -1
        print("Field %d (%s) Tsys scans:" % (f, fieldName), fieldTsysScans,
              ", All scans:", fieldAllScans, ", Non-Tsys scans:",
              fieldNonTsysScans)
        scienceLikeScans = []
        for s in fieldNonTsysScans:
            intents = mymsmd.intentsforscan(s)
            good = True
            for i in intents:
                for b in badIntents:
                    if i.startswith(b):
                        good = False
                        break
            if good: scienceLikeScans.append(s)
        powerRefScans = []
        for s in fieldTsysScans:
            minDist = 9999999
            refScan = -1
            for r in scienceLikeScans:
                dist = abs(r - s)
                if dist < minDist:
                    minDist = dist
                    refScan = r
            powerRefScans.append(refScan)
        if verbose:
            print("Field %d (%s) Tsys scans:" % (f, fieldName), fieldTsysScans,
                  ", All scans:", fieldAllScans, ", Non-Tsys scans:",
                  fieldNonTsysScans, ", Non-Tsys science-like scans:",
                  scienceLikeScans)
        for i in range(len(fieldTsysScans)):
            if verbose:
                print("        Tsys scan %3d power reference scan: %3d" %
                      (fieldTsysScans[i], powerRefScans[i]))
            refScans[fieldTsysScans[i]] = powerRefScans[i]
        if verbose:
            print(
                "populating powers corresponding to each Tsys scan on field %d..."
                % (f))
        for i in range(len(fieldTsysScans)):
            refPowers[fieldTsysScans[i]] = []
            for spw in scaleSpws:
                if verbose:
                    print("powerRefScans: ", powerRefScans)
                    print("calling getPower(vis, %d, %d, 10.0, %s)" %
                          (powerRefScans[i], spw,
                           str(powerRefScans[i] < fieldTsysScans[i])))
                p = getPower(vis,
                             powerRefScans[i],
                             spw,
                             10.0,
                             powerRefScans[i] < fieldTsysScans[i],
                             verbose=verbose)
                refPowers[fieldTsysScans[i]].append(p)
            #print "powers to use for Tsys scan %d:"%fieldTsysScans[i], refPowers[fieldTsysScans[i]]
#    print refPowers

    print("tsysTransfer: summarising Tsys table took %.3f seconds" %
          (time.time() - t))

    t = time.time()
    mytb.copy(newTsysTable)
    mytb.close()
    # re-open original table as read-only
    mytb.open(tsysTable)
    mytbNew = taskinit.tbtool()
    mytbNew.open(newTsysTable, nomodify=False)
    print(
        "tsysTransfer: Copying Tsys table from '%s' to '%s' took %.3f seconds"
        % (tsysTable, newTsysTable, time.time() - t))
    anyProcessingNeeded = False

    # Loop over each Tsys scan
    for i in range(len(tsysScans) - 1):
        tsysTime0 = tsysScanTimes[tsysScans[i]]
        tsysTime1 = tsysScanTimes[tsysScans[i + 1]]
        tsysTimeGap = tsysTime1 - tsysTime0
        tsysFields0 = mymsmd.fieldsforscan(tsysScans[i])  # current Tsys scan
        tsysFields1 = mymsmd.fieldsforscan(tsysScans[i + 1])  # next Tsys scan
        # loop over all scans between the current Tsys scan and the next one
        startScan = tsysScans[i] + 1
        stopScan = tsysScans[i + 1]
        #        if finalScan > stopScan and i==len(tsysScans)-1:
        #            print "There are more scans after the final Tsys scan, extending the range of scans accordingly."
        #            stopScan = finalScan
        for scan in range(startScan, stopScan):
            if 'CALIBRATE_POINTING#ON_SOURCE' in mymsmd.intentsforscan(scan):
                continue
            processingNeeded = False
            fields = mymsmd.fieldsforscan(scan)
            times = mymsmd.timesforscan(scan)
            startTime = times[0]
            endTime = times[-1]
            print(
                "Processing scan %d with fields %s, between Tsys scan %d (fields %s) and %d (fields %s)"
                % (scan, str(fields[0]), tsysScans[i], str(
                    tsysFields0[0]), tsysScans[i + 1], str(tsysFields1[0])))
            print(
                "    Scan %d starts %.3f sec after preceding Tsys, and ends %.3f sec before next Tsys"
                % (scan, startTime - tsysTime0, tsysTime1 - endTime))
            # There are a few possible cases to deal with:
            # 1) this was a power reference scan for a Tsys scan, in which case only produce one extra Tsys, at the opposite end of the scan, or none if there are Tsys scans for the same field at both ends
            fieldMatchesPriorTsysField = fieldsMatch(fields, tsysFields0)
            fieldMatchesNextTsysField = fieldsMatch(fields, tsysFields1)
            priorScanIsTsys = scan == tsysScans[i] + 1
            nextScanIsTsys = scan == tsysScans[i + 1] - 1
            bracketingTsysFieldsMatch = fieldsMatch(tsysFields0, tsysFields1)
            scanIsNotRefScan = scan != refScans[
                tsysScans[i]] and scan != refScans[tsysScans[i + 1]]
            if fieldMatchesPriorTsysField and fieldMatchesNextTsysField and priorScanIsTsys and nextScanIsTsys:
                print(
                    "    Nothing needed for scan %d as bracketed immediately by two Tsys scans of same field"
                    % scan)
            # The most common case for wanting to do the transfer: science field bracketed by phase cal, or phase cal without Tsys immediately before/after
            elif bracketingTsysFieldsMatch and (not fieldMatchesPriorTsysField
                                                or scanIsNotRefScan):
                # The two Tsys scans that bracket this scan are taken on the same field;
                # and either this scan is not on the field of the prior Tsys scan, or
                # this scan is not a reference scan
                processingNeeded = True
                priorScanToUse = tsysScans[i]
                nextScanToUse = tsysScans[i + 1]
            elif (not bracketingTsysFieldsMatch
                  and fields[0] in tsysScansOnField.keys()):
                candidateScans = np.array(tsysScansOnField[fields[0]])
                if (scan < candidateScans[0] or scan > candidateScans[-1]):
                    print(
                        "    The bracketing Tsys fields do not match, and there are not two scans to interpolate between."
                    )
                else:
                    processingNeeded = True
                    priorScanToUse = np.max(
                        candidateScans[np.where(candidateScans < scan)])
                    nextScanToUse = np.min(
                        candidateScans[np.where(candidateScans > scan)])
                    print(
                        "    The bracketing Tsys fields do not match, but there are two scans to interpolate between: %d and %d."
                        % (priorScanToUse, nextScanToUse))
            elif (not bracketingTsysFieldsMatch):
                # This section added by Todd for initial phase calibrator scans when Tsys taken on science target only.
                # Not sure what to do yet, though.
                print(
                    "    The bracketing Tsys fields do not match, and Tsys was never taken on this field. No processing will be done."
                )
                if False:
                    processingNeeded = True
                    if i + 1 < len(tsysScans):
                        print(
                            "    Extrapolating from subsequent Tsys scan: %d" %
                            (tsysScans[i + 1]))
                        priorScanToUse = tsysScans[i + 1]
                        nextScanToUse = tsysScans[i + 1]
                    else:
                        print("    Extrapolating from prior Tsys scan: %d" %
                              (tsysScans[i + 1]))
                        priorScanToUse = tsysScans[i]
                        nextScanToUse = tsysScans[i]
            else:
                print(
                    "    This scan arrangement is unexpected.  No processing will be done."
                )
                print("      bracketingTsysFieldsMatch = %s" %
                      bracketingTsysFieldsMatch)
                print("      fieldMatchesPriorTsysField = %s" %
                      fieldMatchesPriorTsysField)
                print("      fieldMatchesNextTsysField = %s" %
                      fieldMatchesNextTsysField)
                print("      priorScanIsTsys = %s" % priorScanIsTsys)
                print("      nextScanIsTsys = %s" % nextScanIsTsys)
                print("      scanIsNotRefScan = %s" % scanIsNotRefScan)
                print("      %s in tsysScansOnField(%s) = %s" %
                      (fields[0], tsysScansOnField.keys(), fields[0]
                       in tsysScansOnField.keys()))
            if processingNeeded:
                anyProcessingNeeded = True
                print(
                    "    For scan %d will generate two Tsys entries for beginning and end of scan, interpolating reference from scans %d and %d"
                    % (scan, priorScanToUse, nextScanToUse))
                for ispw in range(len(scaleSpws)):
                    spw = scaleSpws[ispw]
                    startPower = getPower(vis,
                                          scan,
                                          spw,
                                          10.0,
                                          False,
                                          verbose=verbose)
                    endPower = getPower(vis,
                                        scan,
                                        spw,
                                        10.0,
                                        True,
                                        verbose=verbose)
                    for ant in range(len(tsysAntennas)):
                        tsysSubTab0 = mytb.query(
                            "SCAN_NUMBER==%d AND SPECTRAL_WINDOW_ID==%d AND ANTENNA1==%d"
                            % (priorScanToUse, tsysSpws[ispw], ant))
                        tsysSubTab1 = mytb.query(
                            "SCAN_NUMBER==%d AND SPECTRAL_WINDOW_ID==%d AND ANTENNA1==%d"
                            % (nextScanToUse, tsysSpws[ispw], ant))
                        # sanity check for duplicate entries
                        if tsysSubTab0.nrows() != 1 or tsysSubTab1.nrows(
                        ) != 1:
                            print(
                                "WARNING!!! not one result row for (scan,ant,spw) query in Tsys table. Scan %d: %d rows, Scan %d: %d rows."
                                % (priorScanToUse, tsysSubTab0.nrows(),
                                   nextScanToUse, tsysSubTab1.nrows()))
                        tsys0 = tsysSubTab0.getcell('FPARAM', 0)
                        tsys1 = tsysSubTab1.getcell('FPARAM', 0)
                        tsysSubTab1.close()
                        startTsys = copy.copy(tsys0)
                        endTsys = copy.copy(
                            tsys0
                        )  # just a placeholder, new values will be filled in below
                        startRefPower = refPowers[priorScanToUse]
                        endRefPower = refPowers[nextScanToUse]
                        tsysTime0 = tsysScanTimes[priorScanToUse]
                        tsysTime1 = tsysScanTimes[nextScanToUse]
                        tsysTimeGap = tsysTime1 - tsysTime0
                        for pol in range(len(tsys0)):
                            for chan in range(len(tsys0[pol])):
                                startTsys0 = TsysAfterPowerChange(
                                    startRefPower[ispw][ant][pol],
                                    startPower[ant][0], tsys0[pol][chan])
                                startTsys1 = TsysAfterPowerChange(
                                    endRefPower[ispw][ant][pol],
                                    startPower[ant][0], tsys1[pol][chan])
                                endTsys0 = TsysAfterPowerChange(
                                    startRefPower[ispw][ant][pol],
                                    endPower[ant][0], tsys0[pol][chan])
                                endTsys1 = TsysAfterPowerChange(
                                    endRefPower[ispw][ant][pol],
                                    endPower[ant][0], tsys1[pol][chan])
                                if tsysTimeGap == 0:
                                    startTsys[pol][chan] = startTsys0
                                    endTsys[pol][chan] = endTsys0
                                else:
                                    startTsys[pol][chan] = (
                                        (startTime - tsysTime0) * startTsys1 +
                                        (tsysTime1 - startTime) *
                                        startTsys0) / tsysTimeGap
                                    endTsys[pol][chan] = (
                                        (endTime - tsysTime0) * endTsys1 +
                                        (tsysTime1 - endTime) *
                                        endTsys0) / tsysTimeGap
                                if chan == len(
                                        tsys0[pol]
                                ) / 2 and ant == printAntenna and pol == printPol:
                                    print(
                                        "    ispw=%d spw=%d ant=%d pol=%d chan=%d: TsysBefore: %.1f K, TsysScanStart: %.1f K (interp %.1f,%.1f), TsysScanEnd: %.1f K (interp %.1f,%.1f), TsysAfter: %.1f K"
                                        %
                                        (ispw, spw, ant, pol, chan,
                                         tsys0[pol][chan],
                                         startTsys[pol][chan], startTsys0,
                                         startTsys1, endTsys[pol][chan],
                                         endTsys0, endTsys1, tsys1[pol][chan]))
                        for f in fields:
                            nr = mytbNew.nrows()
                            tsysSubTab0.copyrows(newTsysTable, nrow=1)
                            if verbose:
                                print("setting tsys at row %d" % nr)
                            mytbNew.putcell('FPARAM', nr, startTsys)
                            mytbNew.putcell('TIME', nr, startTime)
                            mytbNew.putcell('FIELD_ID', nr, f)
                            mytbNew.putcell('SCAN_NUMBER', nr, scan)
                            nr = mytbNew.nrows()
                            tsysSubTab0.copyrows(newTsysTable, nrow=1)
                            if verbose:
                                print("setting tsys at row %d" % nr)
                            mytbNew.putcell('FPARAM', nr, endTsys)
                            mytbNew.putcell('TIME', nr, endTime)
                            mytbNew.putcell('FIELD_ID', nr, f)
                            mytbNew.putcell('SCAN_NUMBER', nr, scan)
                        tsysSubTab0.close()
                        # end loop over fields (f)
                    # end loop over antennas (ant)
                # end loop over spws (ispw)
            # end if processingNeeded
        # end loop over scans between tsysScans (scan)
        mytbNew.flush()
    # end loop over Tsys scans
    if not anyProcessingNeeded:
        print(
            "Because no processing was needed the new Tsys table is identical to the original."
        )
    # TODO: These cleanups should be done also on an exception too
    print("Closing tables...")
    mytbNew.unlock()
    mytbNew.close()
    mytbNew.done()
    mymsmd.close()
    mytb.close()
    mytb.done()
Esempio n. 29
0
def tsysNormalize(vis,
                  tsysTable='',
                  newTsysTable='',
                  scaleSpws=[],
                  verbose=False):
    """
    Generate Tsys entries for one field from other fields, using autocorr
    (linear!) or SQLD data to determine the change in Tsys.
    Inputs:
     vis          the MS
     tsysTable:  the tsys caltable (default = <vis>.tsys)
     newTsysTable:  the new tsys caltable to create (default = <tsysTable>_normalized)
    """

    # intents likely to imply different attenuations or tuning to science-like
    # scans that we are applying Tsys to.
    print("Entered")
    badIntents = [
        'CALIBRATE_POINTING', 'CALIBRATE_FOCUS', 'CALIBRATE_SIDEBAND_RATIO',
        'CALIBRATE_ATMOSPHERE'
    ]
    if (tsysTable == ''):
        tsysTable = vis + '.tsys'
    if (not os.path.exists(tsysTable)):
        print("Cannot find Tsys table: ", tsysTable)
        return
    if (not os.path.exists(vis)):
        print("Cannot find measurement set: ", vis)
        return

    t = time.time()
    mytb = taskinit.tbtool()
    mymsmd = taskinit.msmdtool()
    mytb.open(tsysTable, nomodify=False)
    mymsmd.open(vis)
    print("tsysNormalize: initial setup took %.3f seconds" % (time.time() - t))

    # For convenience squish the useful columns into unique lists
    t = time.time()
    tsysSpws = pb.unique(mytb.getcol("SPECTRAL_WINDOW_ID"))
    tsysScans = pb.unique(mytb.getcol("SCAN_NUMBER"))
    tsysTimes = pb.unique(mytb.getcol("TIME"))
    tsysFields = pb.unique(mytb.getcol("FIELD_ID"))
    tsysAntennas = pb.unique(mytb.getcol("ANTENNA1"))
    if type(scaleSpws) == str:
        scaleSpws = [int(i) for i in scaleSpws.split(',')]
    if len(scaleSpws) < len(tsysSpws):
        scaleSpws = []
        for tsysSpw in tsysSpws:
            scaleSpws.append(scienceSpwForTsysSpw(mymsmd, tsysSpw))
        print("Identified autocorrelation spws to use: ", scaleSpws)
    print("Tsys Spws (%d):" % len(tsysSpws), tsysSpws)
    print("Tsys Scans (%d):" % len(tsysScans), tsysScans)
    print("Tsys Times (%d):" % len(tsysTimes), tsysTimes)
    print("Tsys Fields (%d):" % len(tsysFields), tsysFields)
    print("Tsys Antennas (%d):" % len(tsysAntennas), tsysAntennas)

    # Gather the power levels to use in the normalization process
    refPowers = {}
    refScans = {}
    for f in tsysFields:
        scanFieldsTab = mytb.query('FIELD_ID==%d' % f)
        fieldTsysScans = pb.unique(scanFieldsTab.getcol("SCAN_NUMBER"))
        scanFieldsTab.close()
        fieldAllScans = mymsmd.scansforfield(f)
        fieldNonTsysScans = [
            x for x in fieldAllScans if x not in fieldTsysScans
        ]
        fieldName = mymsmd.namesforfields(f)[0]
        if (len(fieldNonTsysScans) < 1):
            # Then there is no non-tsys scan for this field, e.g. which can happen in a mosaic where the Tsys scan has a different field ID,
            # but in this case the field name will have other scans with different field IDs, so revert to using field names.  Using field
            # names might work from the outset, but I have not tried it.
            fieldAllScans = mymsmd.scansforfield(fieldName)
            fieldNonTsysScans = [
                x for x in fieldAllScans if x not in fieldTsysScans
            ]
            if (len(fieldNonTsysScans) < 1):
                print(
                    "****** This field (id=%d, name=%s) appears to have no non-Tsys-like-scans, and thus cannot be normalized."
                    % (f, fieldName))
                return -1
        scienceLikeScans = []
        for s in fieldNonTsysScans:
            intents = mymsmd.intentsforscan(s)
            good = True
            for i in intents:
                for b in badIntents:
                    if i.startswith(b):
                        good = False
                        break
            if good: scienceLikeScans.append(s)
        powerRefScans = []
        for s in fieldTsysScans:
            minDist = 9999999
            refScan = -1
            for r in scienceLikeScans:
                dist = abs(r - s)
                if dist < minDist:
                    minDist = dist
                    refScan = r
            powerRefScans.append(refScan)
        print("Field %d (%s) Tsys scans:" % (f, fieldname), fieldTsysScans,
              ", All scans:", fieldAllScans, ", Non-Tsys scans:",
              fieldNonTsysScans, ", Non-Tsys science-like scans:",
              scienceLikeScans)
        for i in range(len(fieldTsysScans)):
            print("        Tsys scan %3d power reference scan: %3d" %
                  (fieldTsysScans[i], powerRefScans[i]))
            refScans[fieldTsysScans[i]] = powerRefScans[i]
        if verbose:
            print(
                "populating powers corresponding to each Tsys scan on field %d..."
                % (f))
        for i in range(len(fieldTsysScans)):
            refPowers[fieldTsysScans[i]] = []
            for spw in scaleSpws:
                if verbose:
                    print("calling getPower(vis, %d, %d, 10.0, %s)" %
                          (powerRefScans[i], spw,
                           str(powerRefScans[i] < fieldTsysScans[i])))
                p = getPower(vis,
                             powerRefScans[i],
                             spw,
                             10.0,
                             powerRefScans[i] < fieldTsysScans[i],
                             verbose=verbose)
                refPowers[fieldTsysScans[i]].append(p)
            if verbose:
                print("powers to use for Tsys scan %d:" % fieldTsysScans[i],
                      refPowers[fieldTsysScans[i]])
    if verbose: print(refPowers)

    print("tsysNormalize: summarising Tsys table took %.3f seconds" %
          (time.time() - t))
    t = time.time()

    # Now copy the original Tsys caltable and update all the values in the new one.
    if (newTsysTable == ''):
        newTsysTable = tsysTable + '_normalized'
    if (os.path.exists(newTsysTable)):
        shutil.rmtree(newTsysTable)
    mytb.copy(newTsysTable)
    mytb.close()
    mytb.open(newTsysTable, nomodify=False)
    startRefPower = refPowers[tsysScans[0]]
    for i in range(1, len(tsysScans)):
        # need to adjust each successive Tsys
        refPower = refPowers[tsysScans[i]]
        for ispw in range(len(tsysSpws)):
            spw = tsysSpws[ispw]
            for ant in range(len(tsysAntennas)):
                tsysSubTab1 = mytb.query(
                    "SCAN_NUMBER==%d AND SPECTRAL_WINDOW_ID==%d AND ANTENNA1==%d"
                    % (tsysScans[i], tsysSpws[ispw], ant))
                tsys1 = tsysSubTab1.getcell('FPARAM', 0)
                newTsys = tsysSubTab1.getcell('FPARAM', 0)
                for pol in range(len(tsys1)):
                    for chan in range(len(tsys1[pol])):
                        a = TsysAfterPowerChange(
                            refPowers[tsysScans[i]][ispw][ant][pol],
                            startRefPower[ispw][ant][pol], tsys1[pol][chan])
                        newTsys[pol][chan] = a
                    print("Scan %2d spw %2d pol %d mean %.1f --> %.1f" %
                          (tsysScans[i], spw, pol, np.mean(
                              tsys1[pol]), np.mean(newTsys[pol])))
                tsysSubTab1.putcell('FPARAM', 0, newTsys)
                tsysSubTab1.close()
    mymsmd.close()
    mytb.close()
def plot_weight_density(vis, spw=0, field='', nbins=50, bins=None, clear=False,
                        ignore_flags=False, representative_channel=None,
                        **kwargs):
    """
    Plot the "weight density" vs uvdist: i.e., the sum of the weights in each
    annular bin divided by the area of that bin

    Parameters
    ----------
    vis : str
        The .ms table to plot weights from
    spw : int or str
        The spectral window to plot.  Only one spectral window should be specified.
    field : str
        The field name to plot (if mosaic, make sure it is a name and not a number)
    nbins : int
        The number of bins to create
    bins : None or array
        You can specify specific bins to average the weights in
    ignore_flags : bool
        Ignore the flags in the file.  Flagged data will be plotted alongside
        unflagged.
    representative_channel : None or int
        A specific channel from which to extract flags.  If left as 'None',
        defaults to the mean frequency
    kwargs : dict
        Keyword arguments are passed to msselect (e.g., obsid).  Unfortunately,
        it seems that msselect will happily ignore just about everything it is
        given.
    """

    if hasattr(spw, '__len__'):
        assert len(spw) == 0, "Only one SPW can be plotted."


    mymsmd = msmdtool()
    mymsmd.open(vis)

    reffreq = "{value}{unit}".format(**mymsmd.reffreq(spw)['m0'])
    reffreq = "{0}Hz".format(mymsmd.meanfreq(spw))
    if representative_channel is not None:
        closest_channel = representative_channel
    else:
        closest_channel = np.argmin(np.abs(mymsmd.chanfreqs(spw) - mymsmd.meanfreq(spw)))
    mymsmd.close()

    myms = mstool()

    myms.open(vis)
    myms.selectinit(0)
    selection_dict = dict(field=field) #, spw=reffreq,)
    selection_dict.update(kwargs)
    #print(selection_dict)
    assert myms.msselect(selection_dict), "Data selection has failed"
    #print(myms.msselectedindices())
    # select one "representative" channel out of the SPW (because the weights
    # are per SPW, but the flags are per channel)
    assert myms.selectchannel(start=closest_channel, nchan=1, inc=1, width=1), "Channel selection has failed"
    if ignore_flags:
        columns = ['UVW', 'WEIGHT']
    else:
        columns = ['UVW', 'WEIGHT', 'FLAG']
    datadict=myms.getdata(columns)
    myms.close()
    wt = datadict['weight'].squeeze()
    uvw = datadict['uvw'].squeeze()

    # calculate the UV distance from the uvw array
    uvd = (uvw[:2,:]**2).sum(axis=0)**0.5

    if bins is None:
        bins = np.linspace(uvd.min(), uvd.max(), nbins)


    if not ignore_flags:
        # We have exactly one channel (we forced it above) and the second index
        # should be the channel ID
        # If the flag shape does not conform to this assumption, we're in trouble
        # squeeze just gets rid of all size=1 dimensions
        flags = datadict['flag'].squeeze()

        if flags.shape != wt.shape:
            raise ValueError("Flag shape and weight shape don't match. "
                             "Flag shape: {0}  Weight shape: {1}".format(
                                 flags.shape,wt.shape))

        # set weights to zero because we're adding them (this is obviously not right
        # for many operations, but it is right here!)
        wt[flags] = 0

    # one plot for each polarization
    h_1 = np.histogram(uvd, bins, weights=wt[0,:])
    h_2 = np.histogram(uvd, bins, weights=wt[1,:])

    # plot points at the bin center
    midbins = (bins[:-1] + bins[1:])/2.
    # compute the bin area for division below
    bin_area = (bins[1:]**2-bins[:-1]**2)*np.pi

    if clear:
        pl.clf()
    pl.plot(midbins, h_1[0]/bin_area, drawstyle='steps-mid')
    pl.plot(midbins, h_2[0]/bin_area, drawstyle='steps-mid')
    pl.xlabel("UV Distance")
    pl.ylabel("Sum of weights / annular area")
Esempio n. 31
0
def subvs2(vis=None,
           outputvis=None,
           timerange='',
           spw='',
           mode=None,
           subtime1=None,
           subtime2=None,
           smoothaxis=None,
           smoothtype=None,
           smoothwidth=None,
           splitsel=None,
           reverse=None,
           overwrite=None):
    """Perform vector subtraction for visibilities
    Keyword arguments:
    vis -- Name of input visibility file (MS)
            default: none; example: vis='ngc5921.ms'
    outputvis -- Name of output uv-subtracted visibility file (MS)
                  default: none; example: outputvis='ngc5921_src.ms'
    timerange -- Time range of performing the UV subtraction:
                 default='' means all times.  examples:
                 timerange = 'YYYY/MM/DD/hh:mm:ss~YYYY/MM/DD/hh:mm:ss'
                 timerange = 'hh:mm:ss~hh:mm:ss'
    spw -- Select spectral window/channel.
           default = '' all the spectral channels. Example: spw='0:1~20'
    mode -- operation mode
            default 'linear' 
                mode = 'linear': use a linear fit for the background to be subtracted
                mode = 'lowpass': act as a lowpass filter---smooth the data using different
                        smooth types and window sizes. Can be performed along either time
                        or frequency axis
                mode = 'highpass': act as a highpass filter---smooth the data first, and 
                        subtract the smoothed data from the original. Can be performed along
                        either time or frequency axis
            mode = 'linear' expandable parameters:
                subtime1 -- Time range 1 of the background to be subtracted from the data 
                             default='' means all times.  format:
                             timerange = 'YYYY/MM/DD/hh:mm:ss~YYYY/MM/DD/hh:mm:ss'
                             timerange = 'hh:mm:ss~hh:mm:ss'
                subtime2 -- Time range 2 of the backgroud to be subtracted from the data
                             default='' means all times.  examples:
                             timerange = 'YYYY/MM/DD/hh:mm:ss~YYYY/MM/DD/hh:mm:ss'
                             timerange = 'hh:mm:ss~hh:mm:ss'
            mode = 'lowpass' or 'highpass' expandable parameters:
                smoothaxis -- axis of smooth
                    Default: 'time'
                    smoothaxis = 'time': smooth is along the time axis
                    smoothaxis = 'freq': smooth is along the frequency axis
                smoothtype -- type of the smooth depending on the convolving kernel
                    Default: 'flat'
                    smoothtype = 'flat': convolving kernel is a flat rectangle,
                            equivalent to a boxcar moving smooth
                    smoothtype = 'hanning': Hanning smooth kernel. See numpy.hanning
                    smoothtype = 'hamming': Hamming smooth kernel. See numpy.hamming
                    smoothtype = 'bartlett': Bartlett smooth kernel. See numpy.bartlett
                    smoothtype = 'blackman': Blackman smooth kernel. See numpy.blackman
                smoothwidth -- width of the smooth kernel
                    Default: 5
                    Examples: smoothwidth=5, meaning the width is 5 pixels
    splitsel -- True or False. default = False. If splitsel = False, then the entire input
            measurement set is copied as the output measurement set (outputvis), with 
            background subtracted at selected timerange and spectral channels. 
            If splitsel = True,then only the selected timerange and spectral channels 
            are copied into the output measurement set (outputvis).
    reverse -- True or False. default = False. If reverse = False, then the times indicated
            by subtime1 and/or subtime2 are treated as background and subtracted; If reverse
            = True, then reverse the sign of the background-subtracted data. The option can 
            be used for mapping absorptive structure.
    overwrite -- True or False. default = False. If overwrite = True and
                outputvis already exists, the selected subtime and spw in the 
                output measurment set will be replaced with background subtracted 
                visibilities

    """
    # check the visbility ms
    casalog.post('input parameters:')
    casalog.post('vis: ' + vis)
    casalog.post('outputvis: ' + outputvis)
    casalog.post('smoothaxis: ' + smoothaxis)
    casalog.post('smoothtype: ' + smoothtype)
    casalog.post('smoothwidth: ' + str(smoothwidth))
    if not outputvis or outputvis.isspace():
        raise (ValueError, 'Please specify outputvis')

    if os.path.exists(outputvis):
        if overwrite:
            print(
                "The already existing output measurement set will be updated.")
        else:
            raise (ValueError,
                   "Output MS %s already exists - will not overwrite." %
                   outputvis)
    else:
        if not splitsel:
            shutil.copytree(vis, outputvis)
        else:
            ms.open(vis, nomodify=True)
            ms.split(outputvis, spw=spw, time=timerange, whichcol='DATA')
            ms.close()

    if timerange and (type(timerange) == str):
        [btimeo, etimeo] = timerange.split('~')
        btimeosec = qa.getvalue(qa.convert(qa.totime(btimeo), 's'))
        etimeosec = qa.getvalue(qa.convert(qa.totime(etimeo), 's'))
        timebinosec = etimeosec - btimeosec
        if timebinosec < 0:
            raise Exception(
                'Negative timebin! Please check the "timerange" parameter.')
        casalog.post('Selected timerange: ' + timerange +
                     ' as the time for UV subtraction.')
    else:
        casalog.post(
            'Output timerange not specified, using the entire timerange')

    if spw and (type(spw) == str):
        spwlist = spw.split(';')
    else:
        casalog.post('spw not specified, use all frequency channels')

    # read the output data
    datams = mstool()
    datams.open(outputvis, nomodify=False)
    datamsmd = msmdtool()
    datamsmd.open(outputvis)
    spwinfod = datams.getspectralwindowinfo()
    spwinfok = spwinfod.keys()
    spwinfok.sort(key=int)
    spwinfol = [spwinfod[k] for k in spwinfok]
    for s, spi in enumerate(spwinfol):
        print('processing spectral window {}'.format(spi['SpectralWindowId']))
        datams.selectinit(reset=True)
        staql = {'time': '', 'spw': ''}
        if not splitsel:
            # outputvis is identical to input visibility, do the selection
            if timerange and (type(timerange == str)):
                staql['time'] = timerange
            if spw and (type(spw) == str):
                staql['spw'] = spwlist[s]
            if not spw and not timerange:
                # data selection is not made
                print('selecting all spws and times')
                staql['spw'] = str(spi['SpectralWindowId'])
        else:
            # outputvis is splitted, selections have already applied, select all the data
            print('split the selected spws and times')
            staql['spw'] = str(spi['SpectralWindowId'])
        datams.msselect(staql)
        orec = datams.getdata(['data', 'time', 'axis_info'], ifraxis=True)
        npol, nchan, nbl, ntim = orec['data'].shape
        print('dimension of output data', orec['data'].shape)
        casalog.post('Number of baselines: ' + str(nbl))
        casalog.post('Number of spectral channels: ' + str(nchan))
        casalog.post('Number of time pixels: ' + str(ntim))

        try:
            if mode == 'linear':
                # define and check the background time ranges
                if subtime1 and (type(subtime1) == str):
                    [bsubtime1, esubtime1] = subtime1.split('~')
                    bsubtime1sec = qa.getvalue(
                        qa.convert(qa.totime(bsubtime1), 's'))
                    esubtime1sec = qa.getvalue(
                        qa.convert(qa.totime(esubtime1), 's'))
                    timebin1sec = esubtime1sec - bsubtime1sec
                    if timebin1sec < 0:
                        raise Exception(
                            'Negative timebin! Please check the "subtime1" parameter.'
                        )
                    casalog.post('Selected timerange 1: ' + subtime1 +
                                 ' as background for uv subtraction.')
                else:
                    raise Exception(
                        'Please enter at least one timerange as the background'
                    )
                if subtime2 and (type(subtime2) == str):
                    [bsubtime2, esubtime2] = subtime2.split('~')
                    bsubtime2sec = qa.getvalue(
                        qa.convert(qa.totime(bsubtime2), 's'))
                    esubtime2sec = qa.getvalue(
                        qa.convert(qa.totime(esubtime2), 's'))
                    timebin2sec = esubtime2sec - bsubtime2sec
                    if timebin2sec < 0:
                        raise Exception(
                            'Negative timebin! Please check the "subtime2" parameter.'
                        )
                    timebin2 = str(timebin2sec) + 's'
                    casalog.post('Selected timerange 2: ' + subtime2 +
                                 ' as background for uv subtraction.')
                    # plus 1s is to ensure averaging over the entire timerange
                else:
                    casalog.post(
                        'Timerange 2 not selected, using only timerange 1 as background'
                    )

                # Select the background indicated by subtime1
                ms.open(vis, nomodify=True)
                # Select the spw id
                # ms.msselect({'time': subtime1})
                staql0 = {'time': subtime1, 'spw': ''}
                if spw and (type(spw) == str):
                    staql0['spw'] = spwlist[s]
                else:
                    staql0['spw'] = staql['spw']
                ms.msselect(staql0)
                rec1 = ms.getdata(['data', 'time', 'axis_info'], ifraxis=True)
                # print('shape of the frequency matrix ',rec1['axis_info']['freq_axis']['chan_freq'].shape)
                sz1 = rec1['data'].shape
                print('dimension of selected background 1', rec1['data'].shape)
                # the data shape is (n_pol,n_channel,n_baseline,n_time), no need to reshape
                # rec1['data']=rec1['data'].reshape(sz1[0],sz1[1],sz1[2],nspw,sz1[3]/nspw,order='F')
                # print('reshaped rec1 ', rec1['data'].shape)
                rec1avg = np.average(rec1['data'], axis=3)
                casalog.post('Averaging the visibilities in subtime1: ' +
                             subtime1)
                ms.close()
                if subtime2 and (type(subtime2) == str):
                    ms.open(vis, nomodify=True)
                    # Select the spw id
                    staql0 = {'time': subtime2, 'spw': ''}
                    if spw and (type(spw) == str):
                        staql0['spw'] = spwlist[s]
                    else:
                        staql0['spw'] = staql['spw']
                    ms.msselect(staql0)
                    rec2 = ms.getdata(['data', 'time', 'axis_info'],
                                      ifraxis=True)
                    sz2 = rec2['data'].shape
                    print('dimension of selected background 2',
                          rec2['data'].shape)
                    # rec2['data']=rec2['data'].reshape(sz2[0],sz2[1],sz2[2],nspw,sz2[3]/nspw,order='F')
                    # print('reshaped rec1 ', rec2['data'].shape)
                    rec2avg = np.average(rec2['data'], axis=3)
                    ms.close()
                    casalog.post('Averaged the visibilities in subtime2: ' +
                                 subtime2)
                if subtime1 and (not subtime2):
                    casalog.post(
                        'Only "subtime1" is defined, subtracting background defined in subtime1: '
                        + subtime1)
                    t1 = (np.amax(rec1['time']) + np.amin(rec1['time'])) / 2.
                    print('t1: ',
                          qa.time(qa.quantity(t1, 's'), form='ymd', prec=10))
                    for i in range(ntim):
                        orec['data'][:, :, :, i] -= rec1avg
                        if reverse:
                            orec['data'][:, :, :,
                                         i] = -orec['data'][:, :, :, i]
                if subtime1 and subtime2 and (type(subtime2) == str):
                    casalog.post(
                        'Both subtime1 and subtime2 are specified, doing linear interpolation between "subtime1" and "subtime2"'
                    )
                    t1 = (np.amax(rec1['time']) + np.amin(rec1['time'])) / 2.
                    t2 = (np.amax(rec2['time']) + np.amin(rec2['time'])) / 2.
                    touts = orec['time']
                    print('t1: ',
                          qa.time(qa.quantity(t1, 's'), form='ymd', prec=10))
                    print('t2: ',
                          qa.time(qa.quantity(t2, 's'), form='ymd', prec=10))
                    for i in range(ntim):
                        tout = touts[i]
                        if tout > np.amax([t1, t2]):
                            tout = np.amax([t1, t2])
                        elif tout < np.amin([t1, t2]):
                            tout = np.amin([t1, t2])
                        orec['data'][:, :, :, i] -= (rec2avg - rec1avg) * (
                            tout - t1) / (t2 - t1) + rec1avg
                        if reverse:
                            orec['data'][:, :, :,
                                         i] = -orec['data'][:, :, :, i]
            elif mode == 'highpass':
                if smoothtype != 'flat' and smoothtype != 'hanning' and smoothtype != 'hamming' and smoothtype != 'bartlett' and smoothtype != 'blackman':
                    raise Exception('Unknown smoothtype ' + str(smoothtype))
                if smoothaxis == 'time':
                    if smoothwidth <= 0 or smoothwidth >= ntim:
                        raise Exception(
                            'Specified smooth width is <=0 or >= the total number of '
                            + smoothaxis)
                    else:
                        for i in range(orec['data'].shape[0]):
                            for j in range(nchan):
                                for k in range(nbl):
                                    orec['data'][i, j,
                                                 k, :] -= signalsmooth.smooth(
                                                     orec['data'][i, j, k, :],
                                                     smoothwidth, smoothtype)
                if smoothaxis == 'freq':
                    if smoothwidth <= 0 or smoothwidth >= nchan:
                        raise Exception(
                            'Specified smooth width is <=0 or >= the total number of '
                            + smoothaxis)
                    else:
                        for i in range(orec['data'].shape[0]):
                            for j in range(nbl):
                                for k in range(ntim):
                                    orec['data'][i, :, j,
                                                 k] -= signalsmooth.smooth(
                                                     orec['data'][i, :, j, k],
                                                     smoothwidth, smoothtype)
            elif mode == 'lowpass':
                if smoothtype != 'flat' and smoothtype != 'hanning' and smoothtype != 'hamming' and smoothtype != 'bartlett' and smoothtype != 'blackman':
                    raise Exception('Unknown smoothtype ' + str(smoothtype))
                if smoothaxis == 'time':
                    if smoothwidth <= 0 or smoothwidth >= ntim:
                        raise Exception(
                            'Specified smooth width is <=0 or >= the total number of '
                            + smoothaxis)
                    else:
                        for i in range(orec['data'].shape[0]):
                            for j in range(nchan):
                                for k in range(nbl):
                                    orec['data'][i, j,
                                                 k, :] = signalsmooth.smooth(
                                                     orec['data'][i, j, k, :],
                                                     smoothwidth, smoothtype)
                if smoothaxis == 'freq':
                    if smoothwidth <= 0 or smoothwidth >= nchan:
                        raise Exception(
                            'Specified smooth width is <=0 or >= the total number of '
                            + smoothaxis)
                    else:
                        for i in range(orec['data'].shape[0]):
                            for j in range(nbl):
                                for k in range(ntim):
                                    orec['data'][i, :, j,
                                                 k] = signalsmooth.smooth(
                                                     orec['data'][i, :, j, k],
                                                     smoothwidth, smoothtype)
            else:
                raise Exception('Unknown mode' + str(mode))
        except Exception as instance:
            print('*** Error ***', instance)

        # orec['data']=orec['data'].reshape(szo[0],szo[1],szo[2],szo[3],order='F')
        # put the modified data back into the output visibility set
        del orec['time']
        del orec['axis_info']
        # ms.open(outputvis,nomodify=False)
        # if not splitsel:
        # outputvis is identical to input visibility, do the selection
        #    if timerange and (type(timerange==str)):
        #        datams.msselect({'time':timerange})
        #    if spw and (type(spw)==str):
        #        datams.selectinit(datadescid=int(spwid))
        #        nchan=int(echan)-int(bchan)+1
        #        datams.selectchannel(nchan,int(bchan),1,1)
        #    if not spw and not timerange:
        # data selection is not made
        #        datams.selectinit(datadescid=0)
        # else:
        # outputvis is splitted, selections have already applied, select all the data
        #    datams.selectinit(datadescid=0)
        datams.putdata(orec)
    datams.close()
    datamsmd.done()