def test_readAndParse(self): '''flaghelper: compare the read and parse from a file and from a list of strings''' print '' # <startTime>4891227930515540000 <endTime>4891227932453838000 # <startTime>4891228473545856000 <endTime>4891228473731891000 # <startTime>4891226924455911000 <endTime>4891226927502314000 # <startTime>4891228838164987000 <endTime>4891228838418996000 # <startTime>4891228609440808000 <endTime>4891228612489617000 online = [ "antenna='DV03&&*' timerange='2013/11/15/10:25:30.516~2013/11/15/10:25:32.454'", "antenna='DA44&&*' timerange='2013/11/15/10:34:33.546~2013/11/15/10:34:33.732'", "antenna='DA46&&*' timerange='2013/11/15/10:08:44.456~2013/11/15/10:08:47.502'", "antenna='DV09&&*' timerange='2013/11/15/10:18:11.798~2013/11/15/10:18:13.837'", "antenna='DV05&&*' timerange='2013/11/15/10:40:38.165~2013/11/15/10:40:38.419'" ] myinput = "antenna='DV03&&*' timerange='2013/11/15/10:25:30.516~2013/11/15/10:25:32.454'\n"\ "antenna='DA44&&*' timerange='2013/11/15/10:34:33.546~2013/11/15/10:34:33.732'\n"\ "antenna='DA46&&*' timerange='2013/11/15/10:08:44.456~2013/11/15/10:08:47.502'\n"\ "antenna='DV09&&*' timerange='2013/11/15/10:18:11.798~2013/11/15/10:18:13.837'\n"\ "antenna='DV05&&*' timerange='2013/11/15/10:40:38.165~2013/11/15/10:40:38.419'" filename1 = 'flaghelperonline1.txt' create_input(myinput, filename1) dlist1 = fh.readAndParse([filename1]) self.assertEqual(len(dlist1), 5) # Use the list instead of the file dlist2 = fh.readAndParse(online) self.assertListEqual(dlist1, dlist2) # Compare with the original Flag.xml, second row orig_time_start = float(4891228473545856000) * 1.0E-9 orig_time_end = float(4891228473731891000) * 1.0E-9 proc_time = dlist2[1]['timerange'] t0, t1 = proc_time.split('~', 1) startTime = qa.totime(t0)['value'] startTimeSec = float(startTime * 24 * 3600) endTime = qa.totime(t1)['value'] endTimeSec = float(endTime * 24 * 3600) self.assertAlmostEqual(orig_time_start, startTimeSec, places=3) self.assertAlmostEqual(orig_time_end, endTimeSec, places=3)
def test_readAndParseTbuff(self): '''flaghelper: compare the read and parse and apply tbuff''' print '' # MJD in seconds of timeranges are these # <startTime>4891227930515540000 <endTime>4891227932453838000 # <startTime>4891228473545856000 <endTime>4891228473731891000 # <startTime>4891226924455911000 <endTime>4891226927502314000 # <startTime>4891228838164987000 <endTime>4891228838418996000 # <startTime>4891228609440808000 <endTime>4891228612489617000 online = ["antenna='DV03&&*' timerange='2013/11/15/10:25:30.516~2013/11/15/10:25:32.454'", "antenna='DA44&&*' timerange='2013/11/15/10:34:33.546~2013/11/15/10:34:33.732'", "antenna='DA46&&*' timerange='2013/11/15/10:08:44.456~2013/11/15/10:08:47.502'", "antenna='DV09&&*' timerange='2013/11/15/10:18:11.798~2013/11/15/10:18:13.837'", "antenna='DV05&&*' timerange='2013/11/15/10:40:38.165~2013/11/15/10:40:38.419'"] myinput = "antenna='DV03&&*' timerange='2013/11/15/10:25:30.516~2013/11/15/10:25:32.454'\n"\ "antenna='DA44&&*' timerange='2013/11/15/10:34:33.546~2013/11/15/10:34:33.732'\n"\ "antenna='DA46&&*' timerange='2013/11/15/10:08:44.456~2013/11/15/10:08:47.502'\n"\ "antenna='DV09&&*' timerange='2013/11/15/10:18:11.798~2013/11/15/10:18:13.837'\n"\ "antenna='DV05&&*' timerange='2013/11/15/10:40:38.165~2013/11/15/10:40:38.419'" filename1 = 'flaghelperonline2.txt' create_input(myinput, filename1) # First timerange from online before padding origt = timerange='2013/11/15/10:25:30.516~2013/11/15/10:25:32.454' # Apply tbuff to timeranges timebuffer = 1.1 dlist1 = fh.readAndParse([filename1], tbuff=timebuffer) self.assertEqual(len(dlist1), 5) # Get the first padded timerange from output padt = dlist1[0]['timerange'] # Revert the tbuff application manually t0,t1 = padt.split('~',1) startTime = qa.totime(t0)['value'] startTimeSec = float((startTime * 24 * 3600) + timebuffer) startTimeSec = qa.quantity(startTimeSec, 's') paddedT0 = qa.time(startTimeSec,form='ymd',prec=9)[0] # end time endTime = qa.totime(t1)['value'] endTimeSec = float((endTime * 24 * 3600) - timebuffer) endTimeSec = qa.quantity(endTimeSec, 's') paddedT1 = qa.time(endTimeSec,form='ymd',prec=9)[0] newtimerange = paddedT0+'~'+paddedT1 # Compare with the original self.assertEqual(origt, newtimerange) # Compare with original values from Flag.xml xmlt0 = float(4891227930515540000) * 1.0E-9 xmlt1 = float(4891227932453838000) * 1.0E-9 self.assertAlmostEqual(xmlt0, startTimeSec['value'], places=3) self.assertAlmostEqual(xmlt1, endTimeSec['value'], places=3)
def test_readAndParse(self): '''flaghelper: compare the read and parse from a file and from a list of strings''' print '' # <startTime>4891227930515540000 <endTime>4891227932453838000 # <startTime>4891228473545856000 <endTime>4891228473731891000 # <startTime>4891226924455911000 <endTime>4891226927502314000 # <startTime>4891228838164987000 <endTime>4891228838418996000 # <startTime>4891228609440808000 <endTime>4891228612489617000 online = ["antenna='DV03&&*' timerange='2013/11/15/10:25:30.516~2013/11/15/10:25:32.454'", "antenna='DA44&&*' timerange='2013/11/15/10:34:33.546~2013/11/15/10:34:33.732'", "antenna='DA46&&*' timerange='2013/11/15/10:08:44.456~2013/11/15/10:08:47.502'", "antenna='DV09&&*' timerange='2013/11/15/10:18:11.798~2013/11/15/10:18:13.837'", "antenna='DV05&&*' timerange='2013/11/15/10:40:38.165~2013/11/15/10:40:38.419'"] myinput = "antenna='DV03&&*' timerange='2013/11/15/10:25:30.516~2013/11/15/10:25:32.454'\n"\ "antenna='DA44&&*' timerange='2013/11/15/10:34:33.546~2013/11/15/10:34:33.732'\n"\ "antenna='DA46&&*' timerange='2013/11/15/10:08:44.456~2013/11/15/10:08:47.502'\n"\ "antenna='DV09&&*' timerange='2013/11/15/10:18:11.798~2013/11/15/10:18:13.837'\n"\ "antenna='DV05&&*' timerange='2013/11/15/10:40:38.165~2013/11/15/10:40:38.419'" filename1 = 'flaghelperonline1.txt' create_input(myinput, filename1) dlist1 = fh.readAndParse([filename1]) self.assertEqual(len(dlist1), 5) # Use the list instead of the file dlist2 = fh.readAndParse(online) self.assertListEqual(dlist1, dlist2) # Compare with the original Flag.xml, second row orig_time_start = float(4891228473545856000) * 1.0E-9 orig_time_end = float(4891228473731891000) * 1.0E-9 proc_time = dlist2[1]['timerange'] t0,t1 = proc_time.split('~',1) startTime = qa.totime(t0)['value'] startTimeSec = float(startTime * 24 * 3600) endTime = qa.totime(t1)['value'] endTimeSec = float(endTime * 24 * 3600) self.assertAlmostEqual(orig_time_start, startTimeSec, places=3) self.assertAlmostEqual(orig_time_end, endTimeSec, places=3)
def flagdata( vis, mode, autocorr, # mode manual parameter inpfile, # mode list parameters reason, tbuff, spw, # data selection parameters field, antenna, uvrange, timerange, correlation, scan, intent, array, observation, feed, clipminmax, # mode clip parameters datacolumn, clipoutside, channelavg, timeavg, timebin, clipzeros, quackinterval, # mode quack parameters quackmode, quackincrement, tolerance, # mode shadow parameter addantenna, lowerlimit, # mode elevation parameters upperlimit, ntime, # mode tfcrop combinescans, timecutoff, freqcutoff, timefit, freqfit, maxnpieces, flagdimension, usewindowstats, halfwin, extendflags, winsize, # rflag parameters timedev, freqdev, timedevscale, freqdevscale, spectralmax, spectralmin, extendpols, # mode extend growtime, growfreq, growaround, flagneartime, flagnearfreq, minrel, # mode summary maxrel, minabs, maxabs, spwchan, spwcorr, basecnt, fieldcnt, name, action, # run or not the tool display, flagbackup, savepars, # save the current parameters to FLAG_CMD or to a file cmdreason, # reason to save to flag cmd outfile, # output file to save flag commands writeflags # HIDDEN parameter ): # # Task flagdata # Flags data from an MS or calibration table based on data selection in various ways casalog.origin('flagdata') if (action == 'none' or action == '' or action == 'calculate'): flagbackup = False # SMC: moved the flagbackup to before initializing the cluster. # Note that with this change, a flag backup will be created even if # an error happens that prevents the flagger tool from running. if (mode != 'summary' and flagbackup): casalog.post('Backup original flags before applying new flags') fh.backupFlags(aflocal=None, msfile=vis, prename='flagdata') # Set flagbackup to False because only the controller # should create a backup flagbackup = False # Initialize the helper class orig_locals = locals() FHelper = FlagHelper() # Check if vis is a MS, MMS or cal table: # typevis = 1 --> cal table # typevis = 0 --> MS # typevis = 2 --> MMS iscal = False typevis = fh.isCalTable(vis) if typevis == 1: iscal = True # ***************** Input is MMS -- Parallel Processing *********************** if typevis == 2 and action != '' and action != 'none': # Create a temporary input file with .tmp extension. # Use this file for all the processing from now on. if (isinstance(inpfile,str) and inpfile != '') or \ (isinstance(inpfile, list) and os.path.isfile(inpfile[0])): inpfile = FHelper.setupInputFile(inpfile) if inpfile != None: orig_locals['inpfile'] = inpfile if outfile != '': outfile = os.path.abspath(outfile) orig_locals['outfile'] = outfile if isinstance(addantenna, str) and addantenna != '': addantenna = os.path.abspath(addantenna) orig_locals['addantenna'] = addantenna if isinstance(timedev, str) and timedev != '': timedev = os.path.abspath(timedev) orig_locals['timedev'] = timedev if isinstance(freqdev, str) and freqdev != '': freqdev = os.path.abspath(freqdev) orig_locals['freqdev'] = freqdev FHelper.__init__(orig_locals) # For tests only # FHelper.bypassParallelProcessing(1) FHelper.setupCluster('flagdata') # (CAS-4119): Override summary minabs,maxabs,minrel,maxrel # so that it is done after consolidating the summaries # By-pass options to filter summary filterSummary = False if ((mode == 'summary') and ((minrel != 0.0) or (maxrel != 1.0) or (minabs != 0) or (maxabs != -1))): filterSummary = True myms = mstool() myms.open(vis) subMS_list = myms.getreferencedtables() myms.close() if (minrel != 0.0): minreal_dict = create_arg_dict(subMS_list, 0.0) FHelper.override_arg('minrel', minreal_dict) if (maxrel != 1.0): maxrel_dict = create_arg_dict(subMS_list, 1.0) FHelper.override_arg('maxrel', maxrel_dict) if (minabs != 0): minabs_dict = create_arg_dict(subMS_list, 0) FHelper.override_arg('minabs', minabs_dict) if (maxabs != -1): maxabs_dict = create_arg_dict(subMS_list, -1) FHelper.override_arg('maxabs', maxabs_dict) # By-pass options to filter summary if savepars: myms = mstool() myms.open(vis) subMS_list = myms.getreferencedtables() myms.close() savepars_dict = create_arg_dict(subMS_list, False) FHelper.override_arg('savepars', savepars_dict) # Execute the parallel engines retVar = FHelper.go() # In async mode return the job ids if ParallelTaskHelper.getAsyncMode(): return retVar else: # Filter summary at MMS level if (mode == 'summary'): if filterSummary: retVar = filter_summary(retVar, minrel, maxrel, minabs, maxabs) return retVar # Save parameters at MMS level elif savepars: action = 'none' else: return retVar summary_stats = {} # if pCASA.is_mms(vis): # pCASA.execute("flagdata", orig_locals) # return # ***************** Input is a normal MS/cal table **************** # Create local tools aflocal = casac.agentflagger() mslocal = mstool() try: # Verify the ntime value newtime = 0.0 if type(ntime) == float or type(ntime) == int: if ntime <= 0: raise Exception, 'Parameter ntime cannot be < = 0' else: # units are seconds newtime = float(ntime) elif type(ntime) == str: if ntime == 'scan': # iteration time step is a scan newtime = 0.0 else: # read the units from the string qtime = qa.quantity(ntime) if qtime['unit'] == 'min': # convert to seconds qtime = qa.convert(qtime, 's') elif qtime['unit'] == '': qtime['unit'] = 's' # check units if qtime['unit'] == 's': newtime = qtime['value'] else: casalog.post( 'Cannot convert units of ntime. Will use default 0.0s', 'WARN') casalog.post( "New ntime is of type %s and value %s" % (type(newtime), newtime), 'DEBUG') # Open the MS and attach it to the tool if ((type(vis) == str) & (os.path.exists(vis))): aflocal.open(vis, newtime) else: raise Exception, 'Visibility data set not found - please verify the name' # Get the parameters for the mode agent_pars = {} # By default, write flags to the MS writeflags = True # Only the apply action writes to the MS # action=apply --> write to the MS # action=calculate --> do not write to the MS # action='' --> do not run the tool and do not write to the MS if action != 'apply': writeflags = False # Default mode if mode == '' or mode == 'manualflag': mode = 'manual' # Read in the list of commands # Make a dictionary of the input commands. Select by reason if requested flagcmd = {} if mode == 'list': casalog.post('List mode is active') doPadding = True try: # If tbuff is requested, read and Parse if tbuff == 0.0 or tbuff == [] or tbuff == None: doPadding = False if doPadding: casalog.post('Will apply time buffer padding') # inpfile is a file if isinstance(inpfile, str): inpfile = [inpfile] # read in the list and do a simple parsing to apply tbuff flaglist = fh.readAndParse(inpfile, tbuff) else: # inpfile is a file if isinstance(inpfile, str) and os.path.isfile(inpfile): flaglist = fh.readFile(inpfile) nlines = len(flaglist) casalog.post('Read %s command(s) from file: %s' % (nlines, inpfile)) # inpfile is a list of files elif isinstance(inpfile, list) and os.path.isfile( inpfile[0]): flaglist = fh.readFiles(inpfile) # Python list of strings elif isinstance(inpfile, list): flaglist = inpfile else: raise Exception, 'Unsupported input list of flag commands or input file does not exist' # Parse and create a dictionary flagcmd = fh.parseDictionary(flaglist, reason) # Validate the dictionary. # IMPORTANT: if any parameter changes its type, the following # function needs to be updated. The same if any new parameter is # added or removed from the task fh.evaluateFlagParameters(flagcmd, orig_locals) # List of flag commands in dictionary vrows = flagcmd.keys() casalog.post('%s' % flagcmd, 'DEBUG1') except Exception, instance: casalog.post('%s' % instance, 'ERROR') raise Exception, 'Error reading the input list. Make sure the syntax used in the list '\ 'follows the rules given in the inline help of the task.' casalog.post('Selected ' + str(vrows.__len__()) + ' commands from combined input list(s) ') elif mode == 'manual': agent_pars['autocorr'] = autocorr casalog.post('Manual mode is active')
def flagdata(vis, mode, autocorr, # mode manual parameter inpfile, # mode list parameters reason, tbuff, spw, # data selection parameters field, antenna, uvrange, timerange, correlation, scan, intent, array, observation, feed, clipminmax, # mode clip parameters datacolumn, clipoutside, channelavg, chanbin, timeavg, timebin, clipzeros, quackinterval, # mode quack parameters quackmode, quackincrement, tolerance, # mode shadow parameter addantenna, lowerlimit, # mode elevation parameters upperlimit, ntime, # mode tfcrop combinescans, timecutoff, freqcutoff, timefit, freqfit, maxnpieces, flagdimension, usewindowstats, halfwin, extendflags, winsize, # rflag parameters timedev, freqdev, timedevscale, freqdevscale, spectralmax, spectralmin, extendpols, # mode extend growtime, growfreq, growaround, flagneartime, flagnearfreq, minrel, # mode summary maxrel, minabs, maxabs, spwchan, spwcorr, basecnt, fieldcnt, name, action, # run or not the tool display, flagbackup, savepars, # save the current parameters to FLAG_CMD or to a file cmdreason, # reason to save to flag cmd outfile, # output file to save flag commands overwrite, # overwrite the outfile file writeflags # HIDDEN parameter ): # # Task flagdata # Flags data from an MS or calibration table based on data selection in various ways casalog.origin('flagdata') if (action == 'none' or action=='' or action=='calculate'): flagbackup = False # SMC: moved the flagbackup to before initializing the cluster. # Note that with this change, a flag backup will be created even if # an error happens that prevents the flagger tool from running. if (mode != 'summary' and flagbackup): casalog.post('Backup original flags before applying new flags') fh.backupFlags(aflocal=None, msfile=vis, prename='flagdata') # Set flagbackup to False because only the controller # should create a backup flagbackup = False # Initialize the helper class orig_locals = copy.deepcopy(locals()) FHelper = FlagHelper() # Check if vis is a MS, MMS or cal table: # typevis = 1 --> cal table # typevis = 0 --> MS # typevis = 2 --> MMS iscal = False typevis = fh.isCalTable(vis) if typevis == 1: iscal = True # ***************** Input is MMS -- Parallel Processing *********************** if FHelper.isMPIEnabled() and typevis == 2 and action != '' and action != 'none': # Create a temporary input file with .tmp extension. # Use this file for all the processing from now on. if (isinstance(inpfile,str) and inpfile != '') or \ (isinstance(inpfile, list) and os.path.isfile(inpfile[0])): inpfile = FHelper.setupInputFile(inpfile) if inpfile != None: orig_locals['inpfile'] = inpfile if outfile != '': outfile = os.path.abspath(outfile) orig_locals['outfile'] = outfile if isinstance(addantenna, str) and addantenna != '': addantenna = os.path.abspath(addantenna) orig_locals['addantenna'] = addantenna if isinstance(timedev, str) and timedev != '': timedev = os.path.abspath(timedev) orig_locals['timedev'] = timedev if isinstance(freqdev, str) and freqdev != '': freqdev = os.path.abspath(freqdev) orig_locals['freqdev'] = freqdev FHelper.__init__(orig_locals) # For tests only # FHelper.bypassParallelProcessing(1) FHelper.setupCluster('flagdata') # (CAS-4119): Override summary minabs,maxabs,minrel,maxrel # so that it is done after consolidating the summaries # By-pass options to filter summary filterSummary = False if ((mode == 'summary') and ((minrel != 0.0) or (maxrel != 1.0) or (minabs != 0) or (maxabs != -1))): filterSummary = True myms = mstool() myms.open(vis) subMS_list = myms.getreferencedtables() myms.close() if (minrel != 0.0): minreal_dict = create_arg_dict(subMS_list,0.0) FHelper.override_arg('minrel',minreal_dict) if (maxrel != 1.0): maxrel_dict = create_arg_dict(subMS_list,1.0) FHelper.override_arg('maxrel',maxrel_dict) if (minabs != 0): minabs_dict = create_arg_dict(subMS_list,0) FHelper.override_arg('minabs',minabs_dict) if (maxabs != -1): maxabs_dict = create_arg_dict(subMS_list,-1) FHelper.override_arg('maxabs',maxabs_dict) # By-pass options to filter summary if savepars: myms = mstool() myms.open(vis) subMS_list = myms.getreferencedtables() myms.close() savepars_dict = create_arg_dict(subMS_list,False) FHelper.override_arg('savepars',savepars_dict) # Execute the parallel engines retVar = FHelper.go() # In async mode return the job ids if ParallelTaskHelper.getAsyncMode(): return retVar else: # Filter summary at MMS level if (mode == 'summary'): if filterSummary: retVar = filter_summary(retVar,minrel,maxrel,minabs,maxabs) return retVar # Save parameters at MMS level elif savepars: action = 'none' else: return retVar summary_stats={}; # ***************** Input is a normal MS/cal table **************** # Create local tools aflocal = casac.agentflagger() mslocal = mstool() try: # Verify the ntime value newtime = 0.0 if type(ntime) == float or type(ntime) == int: if ntime <= 0: raise Exception, 'Parameter ntime cannot be < = 0' else: # units are seconds newtime = float(ntime) elif type(ntime) == str: if ntime == 'scan': # iteration time step is a scan newtime = 0.0 else: # read the units from the string qtime = qa.quantity(ntime) if qtime['unit'] == 'min': # convert to seconds qtime = qa.convert(qtime, 's') elif qtime['unit'] == '': qtime['unit'] = 's' # check units if qtime['unit'] == 's': newtime = qtime['value'] else: casalog.post('Cannot convert units of ntime. Will use default 0.0s', 'WARN') casalog.post("New ntime is of type %s and value %s"%(type(newtime),newtime), 'DEBUG') # Open the MS and attach it to the tool if ((type(vis) == str) & (os.path.exists(vis))): aflocal.open(vis, newtime) else: raise Exception, 'Visibility data set not found - please verify the name' # Get the parameters for the mode agent_pars = {} # By default, write flags to the MS writeflags = True # Only the apply action writes to the MS # action=apply --> write to the MS # action=calculate --> do not write to the MS # action='' --> do not run the tool and do not write to the MS if action != 'apply': writeflags = False # Default mode if mode == '' or mode == 'manualflag': mode = 'manual' # Read in the list of commands # Make a dictionary of the input commands. Select by reason if requested flagcmd = {} if mode == 'list': casalog.post('List mode is active') doPadding = True try: # If tbuff is requested, read and Parse if tbuff == 0.0 or tbuff == [] or tbuff == None: doPadding = False if doPadding: casalog.post('Will apply time buffer padding') # inpfile is a file if isinstance(inpfile, str): inpfile = [inpfile] # read in the list and do a simple parsing to apply tbuff flaglist = fh.readAndParse(inpfile, tbuff) else: # inpfile is a file if isinstance(inpfile, str) and os.path.isfile(inpfile): flaglist = fh.readFile(inpfile) nlines = len(flaglist) casalog.post('Read %s command(s) from file: %s'%(nlines, inpfile)) # inpfile is a list of files elif isinstance(inpfile, list) and os.path.isfile(inpfile[0]): flaglist = fh.readFiles(inpfile) # Python list of strings elif isinstance(inpfile, list): flaglist = inpfile else: raise Exception, 'Unsupported input list of flag commands or input file does not exist' # Parse and create a dictionary flagcmd = fh.parseDictionary(flaglist, reason) # Validate the dictionary. # IMPORTANT: if any parameter changes its type, the following # function needs to be updated. The same if any new parameter is # added or removed from the task fh.evaluateFlagParameters(flagcmd,orig_locals) # List of flag commands in dictionary vrows = flagcmd.keys() casalog.post('%s'%flagcmd,'DEBUG1') except Exception, instance: casalog.post('%s'%instance,'ERROR') raise Exception, 'Error reading the input list. Make sure the syntax used in the list '\ 'follows the rules given in the inline help of the task.' casalog.post('Selected ' + str(vrows.__len__()) + ' commands from combined input list(s) ') elif mode == 'manual': agent_pars['autocorr'] = autocorr casalog.post('Manual mode is active')
def test_readAndParseIrregularTbuff(self): '''flaghelper: compare the read and parse and apply of irregular tbuff''' print '' # MJD in seconds of timeranges are these # <startTime>4891227930515540000 <endTime>4891227932453838000 # <startTime>4891228473545856000 <endTime>4891228473731891000 # <startTime>4891226924455911000 <endTime>4891226927502314000 # <startTime>4891228838164987000 <endTime>4891228838418996000 # <startTime>4891228609440808000 <endTime>4891228612489617000 online = ["antenna='DV03&&*' timerange='2013/11/15/10:25:30.516~2013/11/15/10:25:32.454'", "antenna='DA44&&*' timerange='2013/11/15/10:34:33.546~2013/11/15/10:34:33.732'", "antenna='DA46&&*' timerange='2013/11/15/10:08:44.456~2013/11/15/10:08:47.502'", "antenna='DV09&&*' timerange='2013/11/15/10:18:11.798~2013/11/15/10:18:13.837'", "antenna='DV05&&*' timerange='2013/11/15/10:40:38.165~2013/11/15/10:40:38.419'"] myinput = "antenna='DV03&&*' timerange='2013/11/15/10:25:30.516~2013/11/15/10:25:32.454'\n"\ "antenna='DA44&&*' timerange='2013/11/15/10:34:33.546~2013/11/15/10:34:33.732'\n"\ "antenna='DA46&&*' timerange='2013/11/15/10:08:44.456~2013/11/15/10:08:47.502'\n"\ "antenna='DV09&&*' timerange='2013/11/15/10:18:11.798~2013/11/15/10:18:13.837'\n"\ "antenna='DV05&&*' timerange='2013/11/15/10:40:38.165~2013/11/15/10:40:38.419'" filename1 = 'flaghelperonline2.txt' create_input(myinput, filename1) # timeranges from online before padding, for comparison later timeranges=[] for cmd in online: a,b = cmd.split(' ') b = b.lstrip('timerange=') timeranges.append(b.strip("'")) # Apply 2 values of tbuff to timeranges timebuffer = [0.4, 0.7] dlist1 = fh.readAndParse([filename1], tbuff=timebuffer) self.assertEqual(len(dlist1), 5) # check the padded time ranges before and after the application n = 0 for cmd in dlist1: padt = cmd['timerange'] # padt = dlist1[0]['timerange'] # Revert the tbuff application manually t0,t1 = padt.split('~',1) startTime = qa.totime(t0)['value'] startTimeSec = float((startTime * 24 * 3600) + timebuffer[0]) startTimeSec = qa.quantity(startTimeSec, 's') paddedT0 = qa.time(startTimeSec,form='ymd',prec=9)[0] # end time endTime = qa.totime(t1)['value'] endTimeSec = float((endTime * 24 * 3600) - timebuffer[1]) endTimeSec = qa.quantity(endTimeSec, 's') paddedT1 = qa.time(endTimeSec,form='ymd',prec=9)[0] newtimerange = paddedT0+'~'+paddedT1 # Compare with the original self.assertEqual(timeranges[n], newtimerange) n += 1
def test_readAndParseIrregularTbuff(self): '''flaghelper: compare the read and parse and apply of irregular tbuff''' print '' # MJD in seconds of timeranges are these # <startTime>4891227930515540000 <endTime>4891227932453838000 # <startTime>4891228473545856000 <endTime>4891228473731891000 # <startTime>4891226924455911000 <endTime>4891226927502314000 # <startTime>4891228838164987000 <endTime>4891228838418996000 # <startTime>4891228609440808000 <endTime>4891228612489617000 online = [ "antenna='DV03&&*' timerange='2013/11/15/10:25:30.516~2013/11/15/10:25:32.454'", "antenna='DA44&&*' timerange='2013/11/15/10:34:33.546~2013/11/15/10:34:33.732'", "antenna='DA46&&*' timerange='2013/11/15/10:08:44.456~2013/11/15/10:08:47.502'", "antenna='DV09&&*' timerange='2013/11/15/10:18:11.798~2013/11/15/10:18:13.837'", "antenna='DV05&&*' timerange='2013/11/15/10:40:38.165~2013/11/15/10:40:38.419'" ] myinput = "antenna='DV03&&*' timerange='2013/11/15/10:25:30.516~2013/11/15/10:25:32.454'\n"\ "antenna='DA44&&*' timerange='2013/11/15/10:34:33.546~2013/11/15/10:34:33.732'\n"\ "antenna='DA46&&*' timerange='2013/11/15/10:08:44.456~2013/11/15/10:08:47.502'\n"\ "antenna='DV09&&*' timerange='2013/11/15/10:18:11.798~2013/11/15/10:18:13.837'\n"\ "antenna='DV05&&*' timerange='2013/11/15/10:40:38.165~2013/11/15/10:40:38.419'" filename1 = 'flaghelperonline2.txt' create_input(myinput, filename1) # timeranges from online before padding, for comparison later timeranges = [] for cmd in online: a, b = cmd.split(' ') b = b.lstrip('timerange=') timeranges.append(b.strip("'")) # Apply 2 values of tbuff to timeranges timebuffer = [0.4, 0.7] dlist1 = fh.readAndParse([filename1], tbuff=timebuffer) self.assertEqual(len(dlist1), 5) # check the padded time ranges before and after the application n = 0 for cmd in dlist1: padt = cmd['timerange'] # padt = dlist1[0]['timerange'] # Revert the tbuff application manually t0, t1 = padt.split('~', 1) startTime = qa.totime(t0)['value'] startTimeSec = float((startTime * 24 * 3600) + timebuffer[0]) startTimeSec = qa.quantity(startTimeSec, 's') paddedT0 = qa.time(startTimeSec, form='ymd', prec=9)[0] # end time endTime = qa.totime(t1)['value'] endTimeSec = float((endTime * 24 * 3600) - timebuffer[1]) endTimeSec = qa.quantity(endTimeSec, 's') paddedT1 = qa.time(endTimeSec, form='ymd', prec=9)[0] newtimerange = paddedT0 + '~' + paddedT1 # Compare with the original self.assertEqual(timeranges[n], newtimerange) n += 1
def test_readAndParseTbuff(self): '''flaghelper: compare the read and parse and apply tbuff''' print '' # MJD in seconds of timeranges are these # <startTime>4891227930515540000 <endTime>4891227932453838000 # <startTime>4891228473545856000 <endTime>4891228473731891000 # <startTime>4891226924455911000 <endTime>4891226927502314000 # <startTime>4891228838164987000 <endTime>4891228838418996000 # <startTime>4891228609440808000 <endTime>4891228612489617000 online = [ "antenna='DV03&&*' timerange='2013/11/15/10:25:30.516~2013/11/15/10:25:32.454'", "antenna='DA44&&*' timerange='2013/11/15/10:34:33.546~2013/11/15/10:34:33.732'", "antenna='DA46&&*' timerange='2013/11/15/10:08:44.456~2013/11/15/10:08:47.502'", "antenna='DV09&&*' timerange='2013/11/15/10:18:11.798~2013/11/15/10:18:13.837'", "antenna='DV05&&*' timerange='2013/11/15/10:40:38.165~2013/11/15/10:40:38.419'" ] myinput = "antenna='DV03&&*' timerange='2013/11/15/10:25:30.516~2013/11/15/10:25:32.454'\n"\ "antenna='DA44&&*' timerange='2013/11/15/10:34:33.546~2013/11/15/10:34:33.732'\n"\ "antenna='DA46&&*' timerange='2013/11/15/10:08:44.456~2013/11/15/10:08:47.502'\n"\ "antenna='DV09&&*' timerange='2013/11/15/10:18:11.798~2013/11/15/10:18:13.837'\n"\ "antenna='DV05&&*' timerange='2013/11/15/10:40:38.165~2013/11/15/10:40:38.419'" filename1 = 'flaghelperonline2.txt' create_input(myinput, filename1) # First timerange from online before padding origt = timerange = '2013/11/15/10:25:30.516~2013/11/15/10:25:32.454' # Apply tbuff to timeranges timebuffer = 1.1 dlist1 = fh.readAndParse([filename1], tbuff=timebuffer) self.assertEqual(len(dlist1), 5) # Get the first padded timerange from output padt = dlist1[0]['timerange'] # Revert the tbuff application manually t0, t1 = padt.split('~', 1) startTime = qa.totime(t0)['value'] startTimeSec = float((startTime * 24 * 3600) + timebuffer) startTimeSec = qa.quantity(startTimeSec, 's') paddedT0 = qa.time(startTimeSec, form='ymd', prec=9)[0] # end time endTime = qa.totime(t1)['value'] endTimeSec = float((endTime * 24 * 3600) - timebuffer) endTimeSec = qa.quantity(endTimeSec, 's') paddedT1 = qa.time(endTimeSec, form='ymd', prec=9)[0] newtimerange = paddedT0 + '~' + paddedT1 # Compare with the original self.assertEqual(origt, newtimerange) # Compare with original values from Flag.xml xmlt0 = float(4891227930515540000) * 1.0E-9 xmlt1 = float(4891227932453838000) * 1.0E-9 self.assertAlmostEqual(xmlt0, startTimeSec['value'], places=3) self.assertAlmostEqual(xmlt1, endTimeSec['value'], places=3)