def test_readAndParse(self):
'''flaghelper: compare the read and parse from a file and from a list of strings'''
print ''
# <startTime>4891227930515540000 <endTime>4891227932453838000
# <startTime>4891228473545856000 <endTime>4891228473731891000
# <startTime>4891226924455911000 <endTime>4891226927502314000
# <startTime>4891228838164987000 <endTime>4891228838418996000
# <startTime>4891228609440808000 <endTime>4891228612489617000
online = [
"antenna='DV03&&*' timerange='2013/11/15/10:25:30.516~2013/11/15/10:25:32.454'",
"antenna='DA44&&*' timerange='2013/11/15/10:34:33.546~2013/11/15/10:34:33.732'",
"antenna='DA46&&*' timerange='2013/11/15/10:08:44.456~2013/11/15/10:08:47.502'",
"antenna='DV09&&*' timerange='2013/11/15/10:18:11.798~2013/11/15/10:18:13.837'",
"antenna='DV05&&*' timerange='2013/11/15/10:40:38.165~2013/11/15/10:40:38.419'"
]
myinput = "antenna='DV03&&*' timerange='2013/11/15/10:25:30.516~2013/11/15/10:25:32.454'\n"\
"antenna='DA44&&*' timerange='2013/11/15/10:34:33.546~2013/11/15/10:34:33.732'\n"\
"antenna='DA46&&*' timerange='2013/11/15/10:08:44.456~2013/11/15/10:08:47.502'\n"\
"antenna='DV09&&*' timerange='2013/11/15/10:18:11.798~2013/11/15/10:18:13.837'\n"\
"antenna='DV05&&*' timerange='2013/11/15/10:40:38.165~2013/11/15/10:40:38.419'"
filename1 = 'flaghelperonline1.txt'
create_input(myinput, filename1)
dlist1 = fh.readAndParse([filename1])
self.assertEqual(len(dlist1), 5)
# Use the list instead of the file
dlist2 = fh.readAndParse(online)
self.assertListEqual(dlist1, dlist2)
# Compare with the original Flag.xml, second row
orig_time_start = float(4891228473545856000) * 1.0E-9
orig_time_end = float(4891228473731891000) * 1.0E-9
proc_time = dlist2[1]['timerange']
t0, t1 = proc_time.split('~', 1)
startTime = qa.totime(t0)['value']
startTimeSec = float(startTime * 24 * 3600)
endTime = qa.totime(t1)['value']
endTimeSec = float(endTime * 24 * 3600)
self.assertAlmostEqual(orig_time_start, startTimeSec, places=3)
self.assertAlmostEqual(orig_time_end, endTimeSec, places=3)
def test_readAndParseTbuff(self):
'''flaghelper: compare the read and parse and apply tbuff'''
print ''
# MJD in seconds of timeranges are these
# <startTime>4891227930515540000 <endTime>4891227932453838000
# <startTime>4891228473545856000 <endTime>4891228473731891000
# <startTime>4891226924455911000 <endTime>4891226927502314000
# <startTime>4891228838164987000 <endTime>4891228838418996000
# <startTime>4891228609440808000 <endTime>4891228612489617000
online = ["antenna='DV03&&*' timerange='2013/11/15/10:25:30.516~2013/11/15/10:25:32.454'",
"antenna='DA44&&*' timerange='2013/11/15/10:34:33.546~2013/11/15/10:34:33.732'",
"antenna='DA46&&*' timerange='2013/11/15/10:08:44.456~2013/11/15/10:08:47.502'",
"antenna='DV09&&*' timerange='2013/11/15/10:18:11.798~2013/11/15/10:18:13.837'",
"antenna='DV05&&*' timerange='2013/11/15/10:40:38.165~2013/11/15/10:40:38.419'"]
myinput = "antenna='DV03&&*' timerange='2013/11/15/10:25:30.516~2013/11/15/10:25:32.454'\n"\
"antenna='DA44&&*' timerange='2013/11/15/10:34:33.546~2013/11/15/10:34:33.732'\n"\
"antenna='DA46&&*' timerange='2013/11/15/10:08:44.456~2013/11/15/10:08:47.502'\n"\
"antenna='DV09&&*' timerange='2013/11/15/10:18:11.798~2013/11/15/10:18:13.837'\n"\
"antenna='DV05&&*' timerange='2013/11/15/10:40:38.165~2013/11/15/10:40:38.419'"
filename1 = 'flaghelperonline2.txt'
create_input(myinput, filename1)
# First timerange from online before padding
origt = timerange='2013/11/15/10:25:30.516~2013/11/15/10:25:32.454'
# Apply tbuff to timeranges
timebuffer = 1.1
dlist1 = fh.readAndParse([filename1], tbuff=timebuffer)
self.assertEqual(len(dlist1), 5)
# Get the first padded timerange from output
padt = dlist1[0]['timerange']
# Revert the tbuff application manually
t0,t1 = padt.split('~',1)
startTime = qa.totime(t0)['value']
startTimeSec = float((startTime * 24 * 3600) + timebuffer)
startTimeSec = qa.quantity(startTimeSec, 's')
paddedT0 = qa.time(startTimeSec,form='ymd',prec=9)[0]
# end time
endTime = qa.totime(t1)['value']
endTimeSec = float((endTime * 24 * 3600) - timebuffer)
endTimeSec = qa.quantity(endTimeSec, 's')
paddedT1 = qa.time(endTimeSec,form='ymd',prec=9)[0]
newtimerange = paddedT0+'~'+paddedT1
# Compare with the original
self.assertEqual(origt, newtimerange)
# Compare with original values from Flag.xml
xmlt0 = float(4891227930515540000) * 1.0E-9
xmlt1 = float(4891227932453838000) * 1.0E-9
self.assertAlmostEqual(xmlt0, startTimeSec['value'], places=3)
self.assertAlmostEqual(xmlt1, endTimeSec['value'], places=3)
def test_readAndParse(self):
'''flaghelper: compare the read and parse from a file and from a list of strings'''
print ''
# <startTime>4891227930515540000 <endTime>4891227932453838000
# <startTime>4891228473545856000 <endTime>4891228473731891000
# <startTime>4891226924455911000 <endTime>4891226927502314000
# <startTime>4891228838164987000 <endTime>4891228838418996000
# <startTime>4891228609440808000 <endTime>4891228612489617000
online = ["antenna='DV03&&*' timerange='2013/11/15/10:25:30.516~2013/11/15/10:25:32.454'",
"antenna='DA44&&*' timerange='2013/11/15/10:34:33.546~2013/11/15/10:34:33.732'",
"antenna='DA46&&*' timerange='2013/11/15/10:08:44.456~2013/11/15/10:08:47.502'",
"antenna='DV09&&*' timerange='2013/11/15/10:18:11.798~2013/11/15/10:18:13.837'",
"antenna='DV05&&*' timerange='2013/11/15/10:40:38.165~2013/11/15/10:40:38.419'"]
myinput = "antenna='DV03&&*' timerange='2013/11/15/10:25:30.516~2013/11/15/10:25:32.454'\n"\
"antenna='DA44&&*' timerange='2013/11/15/10:34:33.546~2013/11/15/10:34:33.732'\n"\
"antenna='DA46&&*' timerange='2013/11/15/10:08:44.456~2013/11/15/10:08:47.502'\n"\
"antenna='DV09&&*' timerange='2013/11/15/10:18:11.798~2013/11/15/10:18:13.837'\n"\
"antenna='DV05&&*' timerange='2013/11/15/10:40:38.165~2013/11/15/10:40:38.419'"
filename1 = 'flaghelperonline1.txt'
create_input(myinput, filename1)
dlist1 = fh.readAndParse([filename1])
self.assertEqual(len(dlist1), 5)
# Use the list instead of the file
dlist2 = fh.readAndParse(online)
self.assertListEqual(dlist1, dlist2)
# Compare with the original Flag.xml, second row
orig_time_start = float(4891228473545856000) * 1.0E-9
orig_time_end = float(4891228473731891000) * 1.0E-9
proc_time = dlist2[1]['timerange']
t0,t1 = proc_time.split('~',1)
startTime = qa.totime(t0)['value']
startTimeSec = float(startTime * 24 * 3600)
endTime = qa.totime(t1)['value']
endTimeSec = float(endTime * 24 * 3600)
self.assertAlmostEqual(orig_time_start, startTimeSec, places=3)
self.assertAlmostEqual(orig_time_end, endTimeSec, places=3)
def flagdata(
vis,
mode,
autocorr, # mode manual parameter
inpfile, # mode list parameters
reason,
tbuff,
spw, # data selection parameters
field,
antenna,
uvrange,
timerange,
correlation,
scan,
intent,
array,
observation,
feed,
clipminmax, # mode clip parameters
datacolumn,
clipoutside,
channelavg,
timeavg,
timebin,
clipzeros,
quackinterval, # mode quack parameters
quackmode,
quackincrement,
tolerance, # mode shadow parameter
addantenna,
lowerlimit, # mode elevation parameters
upperlimit,
ntime, # mode tfcrop
combinescans,
timecutoff,
freqcutoff,
timefit,
freqfit,
maxnpieces,
flagdimension,
usewindowstats,
halfwin,
extendflags,
winsize, # rflag parameters
timedev,
freqdev,
timedevscale,
freqdevscale,
spectralmax,
spectralmin,
extendpols, # mode extend
growtime,
growfreq,
growaround,
flagneartime,
flagnearfreq,
minrel, # mode summary
maxrel,
minabs,
maxabs,
spwchan,
spwcorr,
basecnt,
fieldcnt,
name,
action, # run or not the tool
display,
flagbackup,
savepars, # save the current parameters to FLAG_CMD or to a file
cmdreason, # reason to save to flag cmd
outfile, # output file to save flag commands
writeflags # HIDDEN parameter
):
#
# Task flagdata
# Flags data from an MS or calibration table based on data selection in various ways
casalog.origin('flagdata')
if (action == 'none' or action == '' or action == 'calculate'):
flagbackup = False
# SMC: moved the flagbackup to before initializing the cluster.
# Note that with this change, a flag backup will be created even if
# an error happens that prevents the flagger tool from running.
if (mode != 'summary' and flagbackup):
casalog.post('Backup original flags before applying new flags')
fh.backupFlags(aflocal=None, msfile=vis, prename='flagdata')
# Set flagbackup to False because only the controller
# should create a backup
flagbackup = False
# Initialize the helper class
orig_locals = locals()
FHelper = FlagHelper()
# Check if vis is a MS, MMS or cal table:
# typevis = 1 --> cal table
# typevis = 0 --> MS
# typevis = 2 --> MMS
iscal = False
typevis = fh.isCalTable(vis)
if typevis == 1:
iscal = True
# ***************** Input is MMS -- Parallel Processing ***********************
if typevis == 2 and action != '' and action != 'none':
# Create a temporary input file with .tmp extension.
# Use this file for all the processing from now on.
if (isinstance(inpfile,str) and inpfile != '') or \
(isinstance(inpfile, list) and os.path.isfile(inpfile[0])):
inpfile = FHelper.setupInputFile(inpfile)
if inpfile != None:
orig_locals['inpfile'] = inpfile
if outfile != '':
outfile = os.path.abspath(outfile)
orig_locals['outfile'] = outfile
if isinstance(addantenna, str) and addantenna != '':
addantenna = os.path.abspath(addantenna)
orig_locals['addantenna'] = addantenna
if isinstance(timedev, str) and timedev != '':
timedev = os.path.abspath(timedev)
orig_locals['timedev'] = timedev
if isinstance(freqdev, str) and freqdev != '':
freqdev = os.path.abspath(freqdev)
orig_locals['freqdev'] = freqdev
FHelper.__init__(orig_locals)
# For tests only
# FHelper.bypassParallelProcessing(1)
FHelper.setupCluster('flagdata')
# (CAS-4119): Override summary minabs,maxabs,minrel,maxrel
# so that it is done after consolidating the summaries
# By-pass options to filter summary
filterSummary = False
if ((mode == 'summary') and ((minrel != 0.0) or (maxrel != 1.0) or
(minabs != 0) or (maxabs != -1))):
filterSummary = True
myms = mstool()
myms.open(vis)
subMS_list = myms.getreferencedtables()
myms.close()
if (minrel != 0.0):
minreal_dict = create_arg_dict(subMS_list, 0.0)
FHelper.override_arg('minrel', minreal_dict)
if (maxrel != 1.0):
maxrel_dict = create_arg_dict(subMS_list, 1.0)
FHelper.override_arg('maxrel', maxrel_dict)
if (minabs != 0):
minabs_dict = create_arg_dict(subMS_list, 0)
FHelper.override_arg('minabs', minabs_dict)
if (maxabs != -1):
maxabs_dict = create_arg_dict(subMS_list, -1)
FHelper.override_arg('maxabs', maxabs_dict)
# By-pass options to filter summary
if savepars:
myms = mstool()
myms.open(vis)
subMS_list = myms.getreferencedtables()
myms.close()
savepars_dict = create_arg_dict(subMS_list, False)
FHelper.override_arg('savepars', savepars_dict)
# Execute the parallel engines
retVar = FHelper.go()
# In async mode return the job ids
if ParallelTaskHelper.getAsyncMode():
return retVar
else:
# Filter summary at MMS level
if (mode == 'summary'):
if filterSummary:
retVar = filter_summary(retVar, minrel, maxrel, minabs,
maxabs)
return retVar
# Save parameters at MMS level
elif savepars:
action = 'none'
else:
return retVar
summary_stats = {}
# if pCASA.is_mms(vis):
# pCASA.execute("flagdata", orig_locals)
# return
# ***************** Input is a normal MS/cal table ****************
# Create local tools
aflocal = casac.agentflagger()
mslocal = mstool()
try:
# Verify the ntime value
newtime = 0.0
if type(ntime) == float or type(ntime) == int:
if ntime <= 0:
raise Exception, 'Parameter ntime cannot be < = 0'
else:
# units are seconds
newtime = float(ntime)
elif type(ntime) == str:
if ntime == 'scan':
# iteration time step is a scan
newtime = 0.0
else:
# read the units from the string
qtime = qa.quantity(ntime)
if qtime['unit'] == 'min':
# convert to seconds
qtime = qa.convert(qtime, 's')
elif qtime['unit'] == '':
qtime['unit'] = 's'
# check units
if qtime['unit'] == 's':
newtime = qtime['value']
else:
casalog.post(
'Cannot convert units of ntime. Will use default 0.0s',
'WARN')
casalog.post(
"New ntime is of type %s and value %s" % (type(newtime), newtime),
'DEBUG')
# Open the MS and attach it to the tool
if ((type(vis) == str) & (os.path.exists(vis))):
aflocal.open(vis, newtime)
else:
raise Exception, 'Visibility data set not found - please verify the name'
# Get the parameters for the mode
agent_pars = {}
# By default, write flags to the MS
writeflags = True
# Only the apply action writes to the MS
# action=apply --> write to the MS
# action=calculate --> do not write to the MS
# action='' --> do not run the tool and do not write to the MS
if action != 'apply':
writeflags = False
# Default mode
if mode == '' or mode == 'manualflag':
mode = 'manual'
# Read in the list of commands
# Make a dictionary of the input commands. Select by reason if requested
flagcmd = {}
if mode == 'list':
casalog.post('List mode is active')
doPadding = True
try:
# If tbuff is requested, read and Parse
if tbuff == 0.0 or tbuff == [] or tbuff == None:
doPadding = False
if doPadding:
casalog.post('Will apply time buffer padding')
# inpfile is a file
if isinstance(inpfile, str):
inpfile = [inpfile]
# read in the list and do a simple parsing to apply tbuff
flaglist = fh.readAndParse(inpfile, tbuff)
else:
# inpfile is a file
if isinstance(inpfile, str) and os.path.isfile(inpfile):
flaglist = fh.readFile(inpfile)
nlines = len(flaglist)
casalog.post('Read %s command(s) from file: %s' %
(nlines, inpfile))
# inpfile is a list of files
elif isinstance(inpfile, list) and os.path.isfile(
inpfile[0]):
flaglist = fh.readFiles(inpfile)
# Python list of strings
elif isinstance(inpfile, list):
flaglist = inpfile
else:
raise Exception, 'Unsupported input list of flag commands or input file does not exist'
# Parse and create a dictionary
flagcmd = fh.parseDictionary(flaglist, reason)
# Validate the dictionary.
# IMPORTANT: if any parameter changes its type, the following
# function needs to be updated. The same if any new parameter is
# added or removed from the task
fh.evaluateFlagParameters(flagcmd, orig_locals)
# List of flag commands in dictionary
vrows = flagcmd.keys()
casalog.post('%s' % flagcmd, 'DEBUG1')
except Exception, instance:
casalog.post('%s' % instance, 'ERROR')
raise Exception, 'Error reading the input list. Make sure the syntax used in the list '\
'follows the rules given in the inline help of the task.'
casalog.post('Selected ' + str(vrows.__len__()) +
' commands from combined input list(s) ')
elif mode == 'manual':
agent_pars['autocorr'] = autocorr
casalog.post('Manual mode is active')
def flagdata(vis,
mode,
autocorr, # mode manual parameter
inpfile, # mode list parameters
reason,
tbuff,
spw, # data selection parameters
field,
antenna,
uvrange,
timerange,
correlation,
scan,
intent,
array,
observation,
feed,
clipminmax, # mode clip parameters
datacolumn,
clipoutside,
channelavg,
chanbin,
timeavg,
timebin,
clipzeros,
quackinterval, # mode quack parameters
quackmode,
quackincrement,
tolerance, # mode shadow parameter
addantenna,
lowerlimit, # mode elevation parameters
upperlimit,
ntime, # mode tfcrop
combinescans,
timecutoff,
freqcutoff,
timefit,
freqfit,
maxnpieces,
flagdimension,
usewindowstats,
halfwin,
extendflags,
winsize, # rflag parameters
timedev,
freqdev,
timedevscale,
freqdevscale,
spectralmax,
spectralmin,
extendpols, # mode extend
growtime,
growfreq,
growaround,
flagneartime,
flagnearfreq,
minrel, # mode summary
maxrel,
minabs,
maxabs,
spwchan,
spwcorr,
basecnt,
fieldcnt,
name,
action, # run or not the tool
display,
flagbackup,
savepars, # save the current parameters to FLAG_CMD or to a file
cmdreason, # reason to save to flag cmd
outfile, # output file to save flag commands
overwrite, # overwrite the outfile file
writeflags # HIDDEN parameter
):
#
# Task flagdata
# Flags data from an MS or calibration table based on data selection in various ways
casalog.origin('flagdata')
if (action == 'none' or action=='' or action=='calculate'):
flagbackup = False
# SMC: moved the flagbackup to before initializing the cluster.
# Note that with this change, a flag backup will be created even if
# an error happens that prevents the flagger tool from running.
if (mode != 'summary' and flagbackup):
casalog.post('Backup original flags before applying new flags')
fh.backupFlags(aflocal=None, msfile=vis, prename='flagdata')
# Set flagbackup to False because only the controller
# should create a backup
flagbackup = False
# Initialize the helper class
orig_locals = copy.deepcopy(locals())
FHelper = FlagHelper()
# Check if vis is a MS, MMS or cal table:
# typevis = 1 --> cal table
# typevis = 0 --> MS
# typevis = 2 --> MMS
iscal = False
typevis = fh.isCalTable(vis)
if typevis == 1:
iscal = True
# ***************** Input is MMS -- Parallel Processing ***********************
if FHelper.isMPIEnabled() and typevis == 2 and action != '' and action != 'none':
# Create a temporary input file with .tmp extension.
# Use this file for all the processing from now on.
if (isinstance(inpfile,str) and inpfile != '') or \
(isinstance(inpfile, list) and os.path.isfile(inpfile[0])):
inpfile = FHelper.setupInputFile(inpfile)
if inpfile != None:
orig_locals['inpfile'] = inpfile
if outfile != '':
outfile = os.path.abspath(outfile)
orig_locals['outfile'] = outfile
if isinstance(addantenna, str) and addantenna != '':
addantenna = os.path.abspath(addantenna)
orig_locals['addantenna'] = addantenna
if isinstance(timedev, str) and timedev != '':
timedev = os.path.abspath(timedev)
orig_locals['timedev'] = timedev
if isinstance(freqdev, str) and freqdev != '':
freqdev = os.path.abspath(freqdev)
orig_locals['freqdev'] = freqdev
FHelper.__init__(orig_locals)
# For tests only
# FHelper.bypassParallelProcessing(1)
FHelper.setupCluster('flagdata')
# (CAS-4119): Override summary minabs,maxabs,minrel,maxrel
# so that it is done after consolidating the summaries
# By-pass options to filter summary
filterSummary = False
if ((mode == 'summary') and ((minrel != 0.0) or (maxrel != 1.0) or (minabs != 0) or (maxabs != -1))):
filterSummary = True
myms = mstool()
myms.open(vis)
subMS_list = myms.getreferencedtables()
myms.close()
if (minrel != 0.0):
minreal_dict = create_arg_dict(subMS_list,0.0)
FHelper.override_arg('minrel',minreal_dict)
if (maxrel != 1.0):
maxrel_dict = create_arg_dict(subMS_list,1.0)
FHelper.override_arg('maxrel',maxrel_dict)
if (minabs != 0):
minabs_dict = create_arg_dict(subMS_list,0)
FHelper.override_arg('minabs',minabs_dict)
if (maxabs != -1):
maxabs_dict = create_arg_dict(subMS_list,-1)
FHelper.override_arg('maxabs',maxabs_dict)
# By-pass options to filter summary
if savepars:
myms = mstool()
myms.open(vis)
subMS_list = myms.getreferencedtables()
myms.close()
savepars_dict = create_arg_dict(subMS_list,False)
FHelper.override_arg('savepars',savepars_dict)
# Execute the parallel engines
retVar = FHelper.go()
# In async mode return the job ids
if ParallelTaskHelper.getAsyncMode():
return retVar
else:
# Filter summary at MMS level
if (mode == 'summary'):
if filterSummary:
retVar = filter_summary(retVar,minrel,maxrel,minabs,maxabs)
return retVar
# Save parameters at MMS level
elif savepars:
action = 'none'
else:
return retVar
summary_stats={};
# ***************** Input is a normal MS/cal table ****************
# Create local tools
aflocal = casac.agentflagger()
mslocal = mstool()
try:
# Verify the ntime value
newtime = 0.0
if type(ntime) == float or type(ntime) == int:
if ntime <= 0:
raise Exception, 'Parameter ntime cannot be < = 0'
else:
# units are seconds
newtime = float(ntime)
elif type(ntime) == str:
if ntime == 'scan':
# iteration time step is a scan
newtime = 0.0
else:
# read the units from the string
qtime = qa.quantity(ntime)
if qtime['unit'] == 'min':
# convert to seconds
qtime = qa.convert(qtime, 's')
elif qtime['unit'] == '':
qtime['unit'] = 's'
# check units
if qtime['unit'] == 's':
newtime = qtime['value']
else:
casalog.post('Cannot convert units of ntime. Will use default 0.0s', 'WARN')
casalog.post("New ntime is of type %s and value %s"%(type(newtime),newtime), 'DEBUG')
# Open the MS and attach it to the tool
if ((type(vis) == str) & (os.path.exists(vis))):
aflocal.open(vis, newtime)
else:
raise Exception, 'Visibility data set not found - please verify the name'
# Get the parameters for the mode
agent_pars = {}
# By default, write flags to the MS
writeflags = True
# Only the apply action writes to the MS
# action=apply --> write to the MS
# action=calculate --> do not write to the MS
# action='' --> do not run the tool and do not write to the MS
if action != 'apply':
writeflags = False
# Default mode
if mode == '' or mode == 'manualflag':
mode = 'manual'
# Read in the list of commands
# Make a dictionary of the input commands. Select by reason if requested
flagcmd = {}
if mode == 'list':
casalog.post('List mode is active')
doPadding = True
try:
# If tbuff is requested, read and Parse
if tbuff == 0.0 or tbuff == [] or tbuff == None:
doPadding = False
if doPadding:
casalog.post('Will apply time buffer padding')
# inpfile is a file
if isinstance(inpfile, str):
inpfile = [inpfile]
# read in the list and do a simple parsing to apply tbuff
flaglist = fh.readAndParse(inpfile, tbuff)
else:
# inpfile is a file
if isinstance(inpfile, str) and os.path.isfile(inpfile):
flaglist = fh.readFile(inpfile)
nlines = len(flaglist)
casalog.post('Read %s command(s) from file: %s'%(nlines, inpfile))
# inpfile is a list of files
elif isinstance(inpfile, list) and os.path.isfile(inpfile[0]):
flaglist = fh.readFiles(inpfile)
# Python list of strings
elif isinstance(inpfile, list):
flaglist = inpfile
else:
raise Exception, 'Unsupported input list of flag commands or input file does not exist'
# Parse and create a dictionary
flagcmd = fh.parseDictionary(flaglist, reason)
# Validate the dictionary.
# IMPORTANT: if any parameter changes its type, the following
# function needs to be updated. The same if any new parameter is
# added or removed from the task
fh.evaluateFlagParameters(flagcmd,orig_locals)
# List of flag commands in dictionary
vrows = flagcmd.keys()
casalog.post('%s'%flagcmd,'DEBUG1')
except Exception, instance:
casalog.post('%s'%instance,'ERROR')
raise Exception, 'Error reading the input list. Make sure the syntax used in the list '\
'follows the rules given in the inline help of the task.'
casalog.post('Selected ' + str(vrows.__len__())
+ ' commands from combined input list(s) ')
elif mode == 'manual':
agent_pars['autocorr'] = autocorr
casalog.post('Manual mode is active')
def test_readAndParseIrregularTbuff(self):
'''flaghelper: compare the read and parse and apply of irregular tbuff'''
print ''
# MJD in seconds of timeranges are these
# <startTime>4891227930515540000 <endTime>4891227932453838000
# <startTime>4891228473545856000 <endTime>4891228473731891000
# <startTime>4891226924455911000 <endTime>4891226927502314000
# <startTime>4891228838164987000 <endTime>4891228838418996000
# <startTime>4891228609440808000 <endTime>4891228612489617000
online = ["antenna='DV03&&*' timerange='2013/11/15/10:25:30.516~2013/11/15/10:25:32.454'",
"antenna='DA44&&*' timerange='2013/11/15/10:34:33.546~2013/11/15/10:34:33.732'",
"antenna='DA46&&*' timerange='2013/11/15/10:08:44.456~2013/11/15/10:08:47.502'",
"antenna='DV09&&*' timerange='2013/11/15/10:18:11.798~2013/11/15/10:18:13.837'",
"antenna='DV05&&*' timerange='2013/11/15/10:40:38.165~2013/11/15/10:40:38.419'"]
myinput = "antenna='DV03&&*' timerange='2013/11/15/10:25:30.516~2013/11/15/10:25:32.454'\n"\
"antenna='DA44&&*' timerange='2013/11/15/10:34:33.546~2013/11/15/10:34:33.732'\n"\
"antenna='DA46&&*' timerange='2013/11/15/10:08:44.456~2013/11/15/10:08:47.502'\n"\
"antenna='DV09&&*' timerange='2013/11/15/10:18:11.798~2013/11/15/10:18:13.837'\n"\
"antenna='DV05&&*' timerange='2013/11/15/10:40:38.165~2013/11/15/10:40:38.419'"
filename1 = 'flaghelperonline2.txt'
create_input(myinput, filename1)
# timeranges from online before padding, for comparison later
timeranges=[]
for cmd in online:
a,b = cmd.split(' ')
b = b.lstrip('timerange=')
timeranges.append(b.strip("'"))
# Apply 2 values of tbuff to timeranges
timebuffer = [0.4, 0.7]
dlist1 = fh.readAndParse([filename1], tbuff=timebuffer)
self.assertEqual(len(dlist1), 5)
# check the padded time ranges before and after the application
n = 0
for cmd in dlist1:
padt = cmd['timerange']
# padt = dlist1[0]['timerange']
# Revert the tbuff application manually
t0,t1 = padt.split('~',1)
startTime = qa.totime(t0)['value']
startTimeSec = float((startTime * 24 * 3600) + timebuffer[0])
startTimeSec = qa.quantity(startTimeSec, 's')
paddedT0 = qa.time(startTimeSec,form='ymd',prec=9)[0]
# end time
endTime = qa.totime(t1)['value']
endTimeSec = float((endTime * 24 * 3600) - timebuffer[1])
endTimeSec = qa.quantity(endTimeSec, 's')
paddedT1 = qa.time(endTimeSec,form='ymd',prec=9)[0]
newtimerange = paddedT0+'~'+paddedT1
# Compare with the original
self.assertEqual(timeranges[n], newtimerange)
n += 1
def test_readAndParseIrregularTbuff(self):
'''flaghelper: compare the read and parse and apply of irregular tbuff'''
print ''
# MJD in seconds of timeranges are these
# <startTime>4891227930515540000 <endTime>4891227932453838000
# <startTime>4891228473545856000 <endTime>4891228473731891000
# <startTime>4891226924455911000 <endTime>4891226927502314000
# <startTime>4891228838164987000 <endTime>4891228838418996000
# <startTime>4891228609440808000 <endTime>4891228612489617000
online = [
"antenna='DV03&&*' timerange='2013/11/15/10:25:30.516~2013/11/15/10:25:32.454'",
"antenna='DA44&&*' timerange='2013/11/15/10:34:33.546~2013/11/15/10:34:33.732'",
"antenna='DA46&&*' timerange='2013/11/15/10:08:44.456~2013/11/15/10:08:47.502'",
"antenna='DV09&&*' timerange='2013/11/15/10:18:11.798~2013/11/15/10:18:13.837'",
"antenna='DV05&&*' timerange='2013/11/15/10:40:38.165~2013/11/15/10:40:38.419'"
]
myinput = "antenna='DV03&&*' timerange='2013/11/15/10:25:30.516~2013/11/15/10:25:32.454'\n"\
"antenna='DA44&&*' timerange='2013/11/15/10:34:33.546~2013/11/15/10:34:33.732'\n"\
"antenna='DA46&&*' timerange='2013/11/15/10:08:44.456~2013/11/15/10:08:47.502'\n"\
"antenna='DV09&&*' timerange='2013/11/15/10:18:11.798~2013/11/15/10:18:13.837'\n"\
"antenna='DV05&&*' timerange='2013/11/15/10:40:38.165~2013/11/15/10:40:38.419'"
filename1 = 'flaghelperonline2.txt'
create_input(myinput, filename1)
# timeranges from online before padding, for comparison later
timeranges = []
for cmd in online:
a, b = cmd.split(' ')
b = b.lstrip('timerange=')
timeranges.append(b.strip("'"))
# Apply 2 values of tbuff to timeranges
timebuffer = [0.4, 0.7]
dlist1 = fh.readAndParse([filename1], tbuff=timebuffer)
self.assertEqual(len(dlist1), 5)
# check the padded time ranges before and after the application
n = 0
for cmd in dlist1:
padt = cmd['timerange']
# padt = dlist1[0]['timerange']
# Revert the tbuff application manually
t0, t1 = padt.split('~', 1)
startTime = qa.totime(t0)['value']
startTimeSec = float((startTime * 24 * 3600) + timebuffer[0])
startTimeSec = qa.quantity(startTimeSec, 's')
paddedT0 = qa.time(startTimeSec, form='ymd', prec=9)[0]
# end time
endTime = qa.totime(t1)['value']
endTimeSec = float((endTime * 24 * 3600) - timebuffer[1])
endTimeSec = qa.quantity(endTimeSec, 's')
paddedT1 = qa.time(endTimeSec, form='ymd', prec=9)[0]
newtimerange = paddedT0 + '~' + paddedT1
# Compare with the original
self.assertEqual(timeranges[n], newtimerange)
n += 1
def test_readAndParseTbuff(self):
'''flaghelper: compare the read and parse and apply tbuff'''
print ''
# MJD in seconds of timeranges are these
# <startTime>4891227930515540000 <endTime>4891227932453838000
# <startTime>4891228473545856000 <endTime>4891228473731891000
# <startTime>4891226924455911000 <endTime>4891226927502314000
# <startTime>4891228838164987000 <endTime>4891228838418996000
# <startTime>4891228609440808000 <endTime>4891228612489617000
online = [
"antenna='DV03&&*' timerange='2013/11/15/10:25:30.516~2013/11/15/10:25:32.454'",
"antenna='DA44&&*' timerange='2013/11/15/10:34:33.546~2013/11/15/10:34:33.732'",
"antenna='DA46&&*' timerange='2013/11/15/10:08:44.456~2013/11/15/10:08:47.502'",
"antenna='DV09&&*' timerange='2013/11/15/10:18:11.798~2013/11/15/10:18:13.837'",
"antenna='DV05&&*' timerange='2013/11/15/10:40:38.165~2013/11/15/10:40:38.419'"
]
myinput = "antenna='DV03&&*' timerange='2013/11/15/10:25:30.516~2013/11/15/10:25:32.454'\n"\
"antenna='DA44&&*' timerange='2013/11/15/10:34:33.546~2013/11/15/10:34:33.732'\n"\
"antenna='DA46&&*' timerange='2013/11/15/10:08:44.456~2013/11/15/10:08:47.502'\n"\
"antenna='DV09&&*' timerange='2013/11/15/10:18:11.798~2013/11/15/10:18:13.837'\n"\
"antenna='DV05&&*' timerange='2013/11/15/10:40:38.165~2013/11/15/10:40:38.419'"
filename1 = 'flaghelperonline2.txt'
create_input(myinput, filename1)
# First timerange from online before padding
origt = timerange = '2013/11/15/10:25:30.516~2013/11/15/10:25:32.454'
# Apply tbuff to timeranges
timebuffer = 1.1
dlist1 = fh.readAndParse([filename1], tbuff=timebuffer)
self.assertEqual(len(dlist1), 5)
# Get the first padded timerange from output
padt = dlist1[0]['timerange']
# Revert the tbuff application manually
t0, t1 = padt.split('~', 1)
startTime = qa.totime(t0)['value']
startTimeSec = float((startTime * 24 * 3600) + timebuffer)
startTimeSec = qa.quantity(startTimeSec, 's')
paddedT0 = qa.time(startTimeSec, form='ymd', prec=9)[0]
# end time
endTime = qa.totime(t1)['value']
endTimeSec = float((endTime * 24 * 3600) - timebuffer)
endTimeSec = qa.quantity(endTimeSec, 's')
paddedT1 = qa.time(endTimeSec, form='ymd', prec=9)[0]
newtimerange = paddedT0 + '~' + paddedT1
# Compare with the original
self.assertEqual(origt, newtimerange)
# Compare with original values from Flag.xml
xmlt0 = float(4891227930515540000) * 1.0E-9
xmlt1 = float(4891227932453838000) * 1.0E-9
self.assertAlmostEqual(xmlt0, startTimeSec['value'], places=3)
self.assertAlmostEqual(xmlt1, endTimeSec['value'], places=3)
