示例#1
0
 def test_readFile2(self):
     '''flaghelper: read a file from disk with spaces in command'''
     # creat input file
     myinput = "scan='1~3' mode='manual'\n"+"scan=' 5'      mode='manualflag'\n"\
               "antenna='DV04 &&*'"
     filename = 'flaghelper2.txt'
     create_input(myinput, filename)
     
     alist = fh.readFile(filename)
     self.assertEqual(len(alist),3)
示例#2
0
 def test_readFile1(self):
     '''flaghelper: read a file from disk'''
     # creat input file
     myinput = "scan='1~3' mode='manual'\n"+"scan='5' mode='manualflag'\n"\
               "#scan='4'"
     filename = 'flaghelper1.txt'
     create_input(myinput, filename)
     
     alist = fh.readFile(filename)
     self.assertEqual(len(alist),2)
示例#3
0
    def test_readFile2(self):
        '''flaghelper: read a file from disk with spaces in command'''
        # creat input file
        myinput = "scan='1~3' mode='manual'\n"+"scan=' 5'      mode='manualflag'\n"\
                  "antenna='DV04 &&*'"
        filename = 'flaghelper2.txt'
        create_input(myinput, filename)

        alist = fh.readFile(filename)
        self.assertEqual(len(alist), 3)
示例#4
0
    def test_readFile1(self):
        '''flaghelper: read a file from disk'''
        # creat input file
        myinput = "scan='1~3' mode='manual'\n"+"scan='5' mode='manualflag'\n"\
                  "#scan='4'"
        filename = 'flaghelper1.txt'
        create_input(myinput, filename)

        alist = fh.readFile(filename)
        self.assertEqual(len(alist), 2)
示例#5
0
 def test_veryLongOnlineFlags(self):
     '''flaghelper: evaluate a very long list of parameters'''
     from tasks import flagdata
     # Get a big input file
     self.setUp_onlineFlags()
     fparams = flagdata.parameters
     
     alist = fh.readFile(self.inpfile)
     adict = fh.parseDictionary(alist)
     os.system('rm -rf '+self.inpfile)
     
     self.assertTrue(fh.evaluateFlagParameters(adict, fparams))
示例#6
0
    def test_veryLongOnlineFlags(self):
        '''flaghelper: evaluate a very long list of parameters'''
        from tasks import flagdata
        # Get a big input file
        self.setUp_onlineFlags()
        fparams = flagdata.parameters

        alist = fh.readFile(self.inpfile)
        adict = fh.parseDictionary(alist)
        os.system('rm -rf ' + self.inpfile)

        self.assertTrue(fh.evaluateFlagParameters(adict, fparams))
示例#7
0
 def test_parseDictionary(self):
     '''flaghelper: read a file and parse to a dictionary'''
     # creat input file
     myinput = "scan='1~3' mode='manual'\n"\
               "scan=' 5'      mode='manualflag'\n"\
               "antenna='DV04 &&*'\n"\
               "#mode=shadow\n"\
               "antenna='DV01&&*' timerange='2013/11/15/10:25:30.516~2013/11/15/10:25:32.454' reason='ACS_not in place'"
     filename = 'flaghelper4.txt'
     create_input(myinput, filename)
     
     alist = fh.readFile(filename)
     self.assertEqual(len(alist),4)
     
     adict = fh.parseDictionary(alist)
     self.assertEqual(adict[3]['command']['timerange'], '2013/11/15/10:25:30.516~2013/11/15/10:25:32.454')
     self.assertEqual(adict[2]['command']['antenna'], 'DV04 &&*')
示例#8
0
    def test_parseDictionary(self):
        '''flaghelper: read a file and parse to a dictionary'''
        # creat input file
        myinput = "scan='1~3' mode='manual'\n"\
                  "scan=' 5'      mode='manualflag'\n"\
                  "antenna='DV04 &&*'\n"\
                  "#mode=shadow\n"\
                  "antenna='DV01&&*' timerange='2013/11/15/10:25:30.516~2013/11/15/10:25:32.454' reason='ACS_not in place'"
        filename = 'flaghelper4.txt'
        create_input(myinput, filename)

        alist = fh.readFile(filename)
        self.assertEqual(len(alist), 4)

        adict = fh.parseDictionary(alist)
        self.assertEqual(adict[3]['command']['timerange'],
                         '2013/11/15/10:25:30.516~2013/11/15/10:25:32.454')
        self.assertEqual(adict[2]['command']['antenna'], 'DV04 &&*')
示例#9
0
def flagdata(
        vis,
        mode,
        autocorr,  # mode manual parameter
        inpfile,  # mode list parameters
        reason,
        tbuff,
        spw,  # data selection parameters
        field,
        antenna,
        uvrange,
        timerange,
        correlation,
        scan,
        intent,
        array,
        observation,
        feed,
        clipminmax,  # mode clip parameters
        datacolumn,
        clipoutside,
        channelavg,
        timeavg,
        timebin,
        clipzeros,
        quackinterval,  # mode quack parameters
        quackmode,
        quackincrement,
        tolerance,  # mode shadow parameter
        addantenna,
        lowerlimit,  # mode elevation parameters
        upperlimit,
        ntime,  # mode tfcrop
        combinescans,
        timecutoff,
        freqcutoff,
        timefit,
        freqfit,
        maxnpieces,
        flagdimension,
        usewindowstats,
        halfwin,
        extendflags,
        winsize,  # rflag parameters
        timedev,
        freqdev,
        timedevscale,
        freqdevscale,
        spectralmax,
        spectralmin,
        extendpols,  # mode extend
        growtime,
        growfreq,
        growaround,
        flagneartime,
        flagnearfreq,
        minrel,  # mode summary
        maxrel,
        minabs,
        maxabs,
        spwchan,
        spwcorr,
        basecnt,
        fieldcnt,
        name,
        action,  # run or not the tool
        display,
        flagbackup,
        savepars,  # save the current parameters to FLAG_CMD  or to a file
        cmdreason,  # reason to save to flag cmd
        outfile,  # output file to save flag commands
        writeflags  # HIDDEN parameter
):

    #
    # Task flagdata
    #    Flags data from an MS or calibration table based on data selection in various ways

    casalog.origin('flagdata')

    if (action == 'none' or action == '' or action == 'calculate'):
        flagbackup = False

    # SMC: moved the flagbackup to before initializing the cluster.
    # Note that with this change, a flag backup will be created even if
    # an error happens that prevents the flagger tool from running.
    if (mode != 'summary' and flagbackup):
        casalog.post('Backup original flags before applying new flags')
        fh.backupFlags(aflocal=None, msfile=vis, prename='flagdata')
        # Set flagbackup to False because only the controller
        # should create a backup
        flagbackup = False

    # Initialize the helper class
    orig_locals = locals()
    FHelper = FlagHelper()

    # Check if vis is a MS, MMS or cal table:
    # typevis = 1 --> cal table
    # typevis = 0 --> MS
    # typevis = 2 --> MMS
    iscal = False
    typevis = fh.isCalTable(vis)
    if typevis == 1:
        iscal = True

    # ***************** Input is MMS -- Parallel Processing ***********************

    if typevis == 2 and action != '' and action != 'none':

        # Create a temporary input file with .tmp extension.
        # Use this file for all the processing from now on.
        if (isinstance(inpfile,str) and inpfile != '') or \
           (isinstance(inpfile, list) and os.path.isfile(inpfile[0])):
            inpfile = FHelper.setupInputFile(inpfile)
            if inpfile != None:
                orig_locals['inpfile'] = inpfile

        if outfile != '':
            outfile = os.path.abspath(outfile)
            orig_locals['outfile'] = outfile
        if isinstance(addantenna, str) and addantenna != '':
            addantenna = os.path.abspath(addantenna)
            orig_locals['addantenna'] = addantenna
        if isinstance(timedev, str) and timedev != '':
            timedev = os.path.abspath(timedev)
            orig_locals['timedev'] = timedev
        if isinstance(freqdev, str) and freqdev != '':
            freqdev = os.path.abspath(freqdev)
            orig_locals['freqdev'] = freqdev

        FHelper.__init__(orig_locals)

        # For tests only
        #        FHelper.bypassParallelProcessing(1)

        FHelper.setupCluster('flagdata')
        # (CAS-4119): Override summary minabs,maxabs,minrel,maxrel
        # so that it is done after consolidating the summaries

        # By-pass options to filter summary
        filterSummary = False
        if ((mode == 'summary') and ((minrel != 0.0) or (maxrel != 1.0) or
                                     (minabs != 0) or (maxabs != -1))):
            filterSummary = True

            myms = mstool()
            myms.open(vis)
            subMS_list = myms.getreferencedtables()
            myms.close()

            if (minrel != 0.0):
                minreal_dict = create_arg_dict(subMS_list, 0.0)
                FHelper.override_arg('minrel', minreal_dict)
            if (maxrel != 1.0):
                maxrel_dict = create_arg_dict(subMS_list, 1.0)
                FHelper.override_arg('maxrel', maxrel_dict)
            if (minabs != 0):
                minabs_dict = create_arg_dict(subMS_list, 0)
                FHelper.override_arg('minabs', minabs_dict)
            if (maxabs != -1):
                maxabs_dict = create_arg_dict(subMS_list, -1)
                FHelper.override_arg('maxabs', maxabs_dict)

        # By-pass options to filter summary
        if savepars:

            myms = mstool()
            myms.open(vis)
            subMS_list = myms.getreferencedtables()
            myms.close()

            savepars_dict = create_arg_dict(subMS_list, False)
            FHelper.override_arg('savepars', savepars_dict)

        # Execute the parallel engines
        retVar = FHelper.go()

        # In async mode return the job ids
        if ParallelTaskHelper.getAsyncMode():
            return retVar
        else:
            # Filter summary at MMS level
            if (mode == 'summary'):
                if filterSummary:
                    retVar = filter_summary(retVar, minrel, maxrel, minabs,
                                            maxabs)
                return retVar
            # Save parameters at MMS level
            elif savepars:
                action = 'none'
            else:
                return retVar

    summary_stats = {}

    #    if pCASA.is_mms(vis):
    #        pCASA.execute("flagdata", orig_locals)
    #        return

    # ***************** Input is a normal MS/cal table ****************

    # Create local tools
    aflocal = casac.agentflagger()
    mslocal = mstool()

    try:
        # Verify the ntime value
        newtime = 0.0
        if type(ntime) == float or type(ntime) == int:
            if ntime <= 0:
                raise Exception, 'Parameter ntime cannot be < = 0'
            else:
                # units are seconds
                newtime = float(ntime)

        elif type(ntime) == str:
            if ntime == 'scan':
                # iteration time step is a scan
                newtime = 0.0
            else:
                # read the units from the string
                qtime = qa.quantity(ntime)

                if qtime['unit'] == 'min':
                    # convert to seconds
                    qtime = qa.convert(qtime, 's')
                elif qtime['unit'] == '':
                    qtime['unit'] = 's'

                # check units
                if qtime['unit'] == 's':
                    newtime = qtime['value']
                else:
                    casalog.post(
                        'Cannot convert units of ntime. Will use default 0.0s',
                        'WARN')

        casalog.post(
            "New ntime is of type %s and value %s" % (type(newtime), newtime),
            'DEBUG')

        # Open the MS and attach it to the tool
        if ((type(vis) == str) & (os.path.exists(vis))):
            aflocal.open(vis, newtime)
        else:
            raise Exception, 'Visibility data set not found - please verify the name'

        # Get the parameters for the mode
        agent_pars = {}

        # By default, write flags to the MS
        writeflags = True

        # Only the apply action writes to the MS
        # action=apply     --> write to the MS
        # action=calculate --> do not write to the MS
        # action=''        --> do not run the tool and do not write to the MS
        if action != 'apply':
            writeflags = False

        # Default mode
        if mode == '' or mode == 'manualflag':
            mode = 'manual'

        # Read in the list of commands
        # Make a dictionary of the input commands. Select by reason if requested
        flagcmd = {}

        if mode == 'list':
            casalog.post('List mode is active')
            doPadding = True
            try:
                # If tbuff is requested, read and Parse
                if tbuff == 0.0 or tbuff == [] or tbuff == None:
                    doPadding = False

                if doPadding:
                    casalog.post('Will apply time buffer padding')

                    # inpfile is a file
                    if isinstance(inpfile, str):
                        inpfile = [inpfile]

                    # read in the list and do a simple parsing to apply tbuff
                    flaglist = fh.readAndParse(inpfile, tbuff)

                else:
                    # inpfile is a file
                    if isinstance(inpfile, str) and os.path.isfile(inpfile):
                        flaglist = fh.readFile(inpfile)
                        nlines = len(flaglist)
                        casalog.post('Read %s command(s) from file: %s' %
                                     (nlines, inpfile))

                    # inpfile is a list of files
                    elif isinstance(inpfile, list) and os.path.isfile(
                            inpfile[0]):
                        flaglist = fh.readFiles(inpfile)

                    # Python list of strings
                    elif isinstance(inpfile, list):
                        flaglist = inpfile

                    else:
                        raise Exception, 'Unsupported input list of flag commands or input file does not exist'

                # Parse and create a dictionary
                flagcmd = fh.parseDictionary(flaglist, reason)

                # Validate the dictionary.
                # IMPORTANT: if any parameter changes its type, the following
                # function needs to be updated. The same if any new parameter is
                # added or removed from the task
                fh.evaluateFlagParameters(flagcmd, orig_locals)

                # List of flag commands in dictionary
                vrows = flagcmd.keys()

                casalog.post('%s' % flagcmd, 'DEBUG1')

            except Exception, instance:
                casalog.post('%s' % instance, 'ERROR')
                raise Exception, 'Error reading the input list. Make sure the syntax used in the list '\
                                 'follows the rules given in the inline help of the task.'

            casalog.post('Selected ' + str(vrows.__len__()) +
                         ' commands from combined input list(s) ')

        elif mode == 'manual':
            agent_pars['autocorr'] = autocorr
            casalog.post('Manual mode is active')
示例#10
0
def flagdata(vis,
             mode,
             autocorr,      # mode manual parameter
             inpfile,       # mode list parameters
             reason,
             tbuff,
             spw,           # data selection parameters
             field,
             antenna,
             uvrange,
             timerange,
             correlation,
             scan,
             intent,
             array,
             observation,
             feed,
             clipminmax,    # mode clip parameters
             datacolumn,
             clipoutside,
             channelavg,
             chanbin,
             timeavg,
             timebin,
             clipzeros,
             quackinterval, # mode quack parameters
             quackmode,
             quackincrement,
             tolerance,      # mode shadow parameter
             addantenna,
             lowerlimit,    # mode elevation parameters
             upperlimit,
             ntime,         # mode tfcrop
             combinescans,
             timecutoff,    
             freqcutoff,
             timefit,
             freqfit,
             maxnpieces,
             flagdimension,
             usewindowstats,
             halfwin,
             extendflags,
             winsize,    # rflag parameters
             timedev,
             freqdev,
             timedevscale,
             freqdevscale,
             spectralmax,
             spectralmin,
             extendpols,    # mode extend
             growtime,
             growfreq,
             growaround,
             flagneartime,
             flagnearfreq,
             minrel,        # mode summary
             maxrel,
             minabs,
             maxabs,
             spwchan,
             spwcorr,
             basecnt,
             fieldcnt,
             name,
             action,           # run or not the tool
             display,
             flagbackup,
             savepars,      # save the current parameters to FLAG_CMD  or to a file
             cmdreason,     # reason to save to flag cmd
             outfile,       # output file to save flag commands
             overwrite,     # overwrite the outfile file
             writeflags    # HIDDEN parameter
            ):      

    
    #
    # Task flagdata
    #    Flags data from an MS or calibration table based on data selection in various ways
    
    casalog.origin('flagdata')
                    
    if (action == 'none' or action=='' or action=='calculate'):
        flagbackup = False
        
    # SMC: moved the flagbackup to before initializing the cluster.
    # Note that with this change, a flag backup will be created even if
    # an error happens that prevents the flagger tool from running.    
    if (mode != 'summary' and flagbackup):
        casalog.post('Backup original flags before applying new flags')
        fh.backupFlags(aflocal=None, msfile=vis, prename='flagdata')
        # Set flagbackup to False because only the controller
        # should create a backup
        flagbackup = False

    # Initialize the helper class
    orig_locals = copy.deepcopy(locals())
    FHelper = FlagHelper()

    # Check if vis is a MS, MMS or cal table:
    # typevis = 1 --> cal table
    # typevis = 0 --> MS
    # typevis = 2 --> MMS
    iscal = False
    typevis = fh.isCalTable(vis)
    if typevis == 1:
        iscal = True


    # ***************** Input is MMS -- Parallel Processing ***********************   
         
    if FHelper.isMPIEnabled() and typevis == 2 and action != '' and action != 'none':
                            
        # Create a temporary input file with .tmp extension.
        # Use this file for all the processing from now on.
        if (isinstance(inpfile,str) and inpfile != '') or \
           (isinstance(inpfile, list) and os.path.isfile(inpfile[0])):
            inpfile = FHelper.setupInputFile(inpfile)
            if inpfile != None:
                orig_locals['inpfile'] = inpfile
        
        if outfile != '':
            outfile = os.path.abspath(outfile)
            orig_locals['outfile'] = outfile
        if isinstance(addantenna, str) and addantenna != '':
            addantenna = os.path.abspath(addantenna)
            orig_locals['addantenna'] = addantenna
        if isinstance(timedev, str) and timedev != '':
            timedev = os.path.abspath(timedev)
            orig_locals['timedev'] = timedev
        if isinstance(freqdev, str) and freqdev != '':
            freqdev = os.path.abspath(freqdev)
            orig_locals['freqdev'] = freqdev    
    
        FHelper.__init__(orig_locals)
        
        # For tests only
#        FHelper.bypassParallelProcessing(1)

        FHelper.setupCluster('flagdata')
        # (CAS-4119): Override summary minabs,maxabs,minrel,maxrel 
        # so that it is done after consolidating the summaries
        
        # By-pass options to filter summary
        filterSummary = False
        if ((mode == 'summary') and ((minrel != 0.0) or (maxrel != 1.0) or (minabs != 0) or (maxabs != -1))):
            filterSummary = True
            
            myms = mstool()
            myms.open(vis)
            subMS_list = myms.getreferencedtables()
            myms.close()
            
            if (minrel != 0.0):
                minreal_dict = create_arg_dict(subMS_list,0.0)
                FHelper.override_arg('minrel',minreal_dict)
            if (maxrel != 1.0):
                maxrel_dict = create_arg_dict(subMS_list,1.0)
                FHelper.override_arg('maxrel',maxrel_dict)
            if (minabs != 0):
                minabs_dict = create_arg_dict(subMS_list,0)
                FHelper.override_arg('minabs',minabs_dict)
            if (maxabs != -1):
                maxabs_dict = create_arg_dict(subMS_list,-1)
                FHelper.override_arg('maxabs',maxabs_dict)
                
        # By-pass options to filter summary
        if savepars:  
            
            myms = mstool()
            myms.open(vis)
            subMS_list = myms.getreferencedtables()
            myms.close()
            
            savepars_dict = create_arg_dict(subMS_list,False)
            FHelper.override_arg('savepars',savepars_dict)
            
        # Execute the parallel engines
        retVar = FHelper.go()
        
        # In async mode return the job ids
        if ParallelTaskHelper.getAsyncMode():
            return retVar
        else:
            # Filter summary at MMS level
            if (mode == 'summary'):
                if filterSummary:
                    retVar = filter_summary(retVar,minrel,maxrel,minabs,maxabs)
                return retVar
            # Save parameters at MMS level
            elif savepars:
                action = 'none'
            else:
                return retVar
    
    summary_stats={};
    
    
    # ***************** Input is a normal MS/cal table ****************
    
    # Create local tools
    aflocal = casac.agentflagger()
    mslocal = mstool()

    try: 
        # Verify the ntime value
        newtime = 0.0
        if type(ntime) == float or type(ntime) == int:
            if ntime <= 0:
                raise Exception, 'Parameter ntime cannot be < = 0'
            else:
                # units are seconds
                newtime = float(ntime)
        
        elif type(ntime) == str:
            if ntime == 'scan':
                # iteration time step is a scan
                newtime = 0.0
            else:
                # read the units from the string
                qtime = qa.quantity(ntime)
                
                if qtime['unit'] == 'min':
                    # convert to seconds
                    qtime = qa.convert(qtime, 's')
                elif qtime['unit'] == '':
                    qtime['unit'] = 's'
                    
                # check units
                if qtime['unit'] == 's':
                    newtime = qtime['value']
                else:
                    casalog.post('Cannot convert units of ntime. Will use default 0.0s', 'WARN')
                                    
        casalog.post("New ntime is of type %s and value %s"%(type(newtime),newtime), 'DEBUG')
                
        # Open the MS and attach it to the tool
        if ((type(vis) == str) & (os.path.exists(vis))):
            aflocal.open(vis, newtime)
        else:
            raise Exception, 'Visibility data set not found - please verify the name'


        # Get the parameters for the mode
        agent_pars = {}
        
        # By default, write flags to the MS
        writeflags = True
        
        # Only the apply action writes to the MS
        # action=apply     --> write to the MS
        # action=calculate --> do not write to the MS
        # action=''        --> do not run the tool and do not write to the MS
        if action != 'apply':
            writeflags = False
                                         
        # Default mode
        if mode == '' or mode == 'manualflag':
            mode = 'manual'
        
        # Read in the list of commands
        # Make a dictionary of the input commands. Select by reason if requested
        flagcmd = {}
        
        if mode == 'list':
            casalog.post('List mode is active')
            doPadding = True
            try:
                # If tbuff is requested, read and Parse
                if tbuff == 0.0 or tbuff == [] or tbuff == None:
                    doPadding = False
                     
                if doPadding:
                    casalog.post('Will apply time buffer padding')
 
                    # inpfile is a file
                    if isinstance(inpfile, str):
                        inpfile = [inpfile]
                         
                    # read in the list and do a simple parsing to apply tbuff
                    flaglist = fh.readAndParse(inpfile, tbuff)
                     
                else:                    
                    # inpfile is a file
                    if isinstance(inpfile, str) and os.path.isfile(inpfile):
                        flaglist = fh.readFile(inpfile)
                        nlines = len(flaglist)
                        casalog.post('Read %s command(s) from file: %s'%(nlines, inpfile))                              
                         
                    # inpfile is a list of files
                    elif isinstance(inpfile, list) and os.path.isfile(inpfile[0]):
                        flaglist = fh.readFiles(inpfile)
                         
                    # Python list of strings
                    elif isinstance(inpfile, list):                    
                        flaglist = inpfile
                        
                    else:
                        raise Exception, 'Unsupported input list of flag commands or input file does not exist'
                             
                         
                # Parse and create a dictionary
                flagcmd = fh.parseDictionary(flaglist, reason)
                 
                # Validate the dictionary. 
                # IMPORTANT: if any parameter changes its type, the following
                # function needs to be updated. The same if any new parameter is
                # added or removed from the task
                fh.evaluateFlagParameters(flagcmd,orig_locals)
                     
                # List of flag commands in dictionary
                vrows = flagcmd.keys()
 
                casalog.post('%s'%flagcmd,'DEBUG1')
                 
                 
            except Exception, instance:
                casalog.post('%s'%instance,'ERROR')
                raise Exception, 'Error reading the input list. Make sure the syntax used in the list '\
                                 'follows the rules given in the inline help of the task.'

            casalog.post('Selected ' + str(vrows.__len__())
                         + ' commands from combined input list(s) ')
                             
        elif mode == 'manual':
            agent_pars['autocorr'] = autocorr
            casalog.post('Manual mode is active')