def test_readFile2(self):
'''flaghelper: read a file from disk with spaces in command'''
# creat input file
myinput = "scan='1~3' mode='manual'\n"+"scan=' 5' mode='manualflag'\n"\
"antenna='DV04 &&*'"
filename = 'flaghelper2.txt'
create_input(myinput, filename)
alist = fh.readFile(filename)
self.assertEqual(len(alist),3)
def test_readFile1(self):
'''flaghelper: read a file from disk'''
# creat input file
myinput = "scan='1~3' mode='manual'\n"+"scan='5' mode='manualflag'\n"\
"#scan='4'"
filename = 'flaghelper1.txt'
create_input(myinput, filename)
alist = fh.readFile(filename)
self.assertEqual(len(alist),2)
def test_readFile2(self):
'''flaghelper: read a file from disk with spaces in command'''
# creat input file
myinput = "scan='1~3' mode='manual'\n"+"scan=' 5' mode='manualflag'\n"\
"antenna='DV04 &&*'"
filename = 'flaghelper2.txt'
create_input(myinput, filename)
alist = fh.readFile(filename)
self.assertEqual(len(alist), 3)
def test_readFile1(self):
'''flaghelper: read a file from disk'''
# creat input file
myinput = "scan='1~3' mode='manual'\n"+"scan='5' mode='manualflag'\n"\
"#scan='4'"
filename = 'flaghelper1.txt'
create_input(myinput, filename)
alist = fh.readFile(filename)
self.assertEqual(len(alist), 2)
def test_veryLongOnlineFlags(self):
'''flaghelper: evaluate a very long list of parameters'''
from tasks import flagdata
# Get a big input file
self.setUp_onlineFlags()
fparams = flagdata.parameters
alist = fh.readFile(self.inpfile)
adict = fh.parseDictionary(alist)
os.system('rm -rf '+self.inpfile)
self.assertTrue(fh.evaluateFlagParameters(adict, fparams))
def test_veryLongOnlineFlags(self):
'''flaghelper: evaluate a very long list of parameters'''
from tasks import flagdata
# Get a big input file
self.setUp_onlineFlags()
fparams = flagdata.parameters
alist = fh.readFile(self.inpfile)
adict = fh.parseDictionary(alist)
os.system('rm -rf ' + self.inpfile)
self.assertTrue(fh.evaluateFlagParameters(adict, fparams))
def test_parseDictionary(self):
'''flaghelper: read a file and parse to a dictionary'''
# creat input file
myinput = "scan='1~3' mode='manual'\n"\
"scan=' 5' mode='manualflag'\n"\
"antenna='DV04 &&*'\n"\
"#mode=shadow\n"\
"antenna='DV01&&*' timerange='2013/11/15/10:25:30.516~2013/11/15/10:25:32.454' reason='ACS_not in place'"
filename = 'flaghelper4.txt'
create_input(myinput, filename)
alist = fh.readFile(filename)
self.assertEqual(len(alist),4)
adict = fh.parseDictionary(alist)
self.assertEqual(adict[3]['command']['timerange'], '2013/11/15/10:25:30.516~2013/11/15/10:25:32.454')
self.assertEqual(adict[2]['command']['antenna'], 'DV04 &&*')
def test_parseDictionary(self):
'''flaghelper: read a file and parse to a dictionary'''
# creat input file
myinput = "scan='1~3' mode='manual'\n"\
"scan=' 5' mode='manualflag'\n"\
"antenna='DV04 &&*'\n"\
"#mode=shadow\n"\
"antenna='DV01&&*' timerange='2013/11/15/10:25:30.516~2013/11/15/10:25:32.454' reason='ACS_not in place'"
filename = 'flaghelper4.txt'
create_input(myinput, filename)
alist = fh.readFile(filename)
self.assertEqual(len(alist), 4)
adict = fh.parseDictionary(alist)
self.assertEqual(adict[3]['command']['timerange'],
'2013/11/15/10:25:30.516~2013/11/15/10:25:32.454')
self.assertEqual(adict[2]['command']['antenna'], 'DV04 &&*')
def flagdata(
vis,
mode,
autocorr, # mode manual parameter
inpfile, # mode list parameters
reason,
tbuff,
spw, # data selection parameters
field,
antenna,
uvrange,
timerange,
correlation,
scan,
intent,
array,
observation,
feed,
clipminmax, # mode clip parameters
datacolumn,
clipoutside,
channelavg,
timeavg,
timebin,
clipzeros,
quackinterval, # mode quack parameters
quackmode,
quackincrement,
tolerance, # mode shadow parameter
addantenna,
lowerlimit, # mode elevation parameters
upperlimit,
ntime, # mode tfcrop
combinescans,
timecutoff,
freqcutoff,
timefit,
freqfit,
maxnpieces,
flagdimension,
usewindowstats,
halfwin,
extendflags,
winsize, # rflag parameters
timedev,
freqdev,
timedevscale,
freqdevscale,
spectralmax,
spectralmin,
extendpols, # mode extend
growtime,
growfreq,
growaround,
flagneartime,
flagnearfreq,
minrel, # mode summary
maxrel,
minabs,
maxabs,
spwchan,
spwcorr,
basecnt,
fieldcnt,
name,
action, # run or not the tool
display,
flagbackup,
savepars, # save the current parameters to FLAG_CMD or to a file
cmdreason, # reason to save to flag cmd
outfile, # output file to save flag commands
writeflags # HIDDEN parameter
):
#
# Task flagdata
# Flags data from an MS or calibration table based on data selection in various ways
casalog.origin('flagdata')
if (action == 'none' or action == '' or action == 'calculate'):
flagbackup = False
# SMC: moved the flagbackup to before initializing the cluster.
# Note that with this change, a flag backup will be created even if
# an error happens that prevents the flagger tool from running.
if (mode != 'summary' and flagbackup):
casalog.post('Backup original flags before applying new flags')
fh.backupFlags(aflocal=None, msfile=vis, prename='flagdata')
# Set flagbackup to False because only the controller
# should create a backup
flagbackup = False
# Initialize the helper class
orig_locals = locals()
FHelper = FlagHelper()
# Check if vis is a MS, MMS or cal table:
# typevis = 1 --> cal table
# typevis = 0 --> MS
# typevis = 2 --> MMS
iscal = False
typevis = fh.isCalTable(vis)
if typevis == 1:
iscal = True
# ***************** Input is MMS -- Parallel Processing ***********************
if typevis == 2 and action != '' and action != 'none':
# Create a temporary input file with .tmp extension.
# Use this file for all the processing from now on.
if (isinstance(inpfile,str) and inpfile != '') or \
(isinstance(inpfile, list) and os.path.isfile(inpfile[0])):
inpfile = FHelper.setupInputFile(inpfile)
if inpfile != None:
orig_locals['inpfile'] = inpfile
if outfile != '':
outfile = os.path.abspath(outfile)
orig_locals['outfile'] = outfile
if isinstance(addantenna, str) and addantenna != '':
addantenna = os.path.abspath(addantenna)
orig_locals['addantenna'] = addantenna
if isinstance(timedev, str) and timedev != '':
timedev = os.path.abspath(timedev)
orig_locals['timedev'] = timedev
if isinstance(freqdev, str) and freqdev != '':
freqdev = os.path.abspath(freqdev)
orig_locals['freqdev'] = freqdev
FHelper.__init__(orig_locals)
# For tests only
# FHelper.bypassParallelProcessing(1)
FHelper.setupCluster('flagdata')
# (CAS-4119): Override summary minabs,maxabs,minrel,maxrel
# so that it is done after consolidating the summaries
# By-pass options to filter summary
filterSummary = False
if ((mode == 'summary') and ((minrel != 0.0) or (maxrel != 1.0) or
(minabs != 0) or (maxabs != -1))):
filterSummary = True
myms = mstool()
myms.open(vis)
subMS_list = myms.getreferencedtables()
myms.close()
if (minrel != 0.0):
minreal_dict = create_arg_dict(subMS_list, 0.0)
FHelper.override_arg('minrel', minreal_dict)
if (maxrel != 1.0):
maxrel_dict = create_arg_dict(subMS_list, 1.0)
FHelper.override_arg('maxrel', maxrel_dict)
if (minabs != 0):
minabs_dict = create_arg_dict(subMS_list, 0)
FHelper.override_arg('minabs', minabs_dict)
if (maxabs != -1):
maxabs_dict = create_arg_dict(subMS_list, -1)
FHelper.override_arg('maxabs', maxabs_dict)
# By-pass options to filter summary
if savepars:
myms = mstool()
myms.open(vis)
subMS_list = myms.getreferencedtables()
myms.close()
savepars_dict = create_arg_dict(subMS_list, False)
FHelper.override_arg('savepars', savepars_dict)
# Execute the parallel engines
retVar = FHelper.go()
# In async mode return the job ids
if ParallelTaskHelper.getAsyncMode():
return retVar
else:
# Filter summary at MMS level
if (mode == 'summary'):
if filterSummary:
retVar = filter_summary(retVar, minrel, maxrel, minabs,
maxabs)
return retVar
# Save parameters at MMS level
elif savepars:
action = 'none'
else:
return retVar
summary_stats = {}
# if pCASA.is_mms(vis):
# pCASA.execute("flagdata", orig_locals)
# return
# ***************** Input is a normal MS/cal table ****************
# Create local tools
aflocal = casac.agentflagger()
mslocal = mstool()
try:
# Verify the ntime value
newtime = 0.0
if type(ntime) == float or type(ntime) == int:
if ntime <= 0:
raise Exception, 'Parameter ntime cannot be < = 0'
else:
# units are seconds
newtime = float(ntime)
elif type(ntime) == str:
if ntime == 'scan':
# iteration time step is a scan
newtime = 0.0
else:
# read the units from the string
qtime = qa.quantity(ntime)
if qtime['unit'] == 'min':
# convert to seconds
qtime = qa.convert(qtime, 's')
elif qtime['unit'] == '':
qtime['unit'] = 's'
# check units
if qtime['unit'] == 's':
newtime = qtime['value']
else:
casalog.post(
'Cannot convert units of ntime. Will use default 0.0s',
'WARN')
casalog.post(
"New ntime is of type %s and value %s" % (type(newtime), newtime),
'DEBUG')
# Open the MS and attach it to the tool
if ((type(vis) == str) & (os.path.exists(vis))):
aflocal.open(vis, newtime)
else:
raise Exception, 'Visibility data set not found - please verify the name'
# Get the parameters for the mode
agent_pars = {}
# By default, write flags to the MS
writeflags = True
# Only the apply action writes to the MS
# action=apply --> write to the MS
# action=calculate --> do not write to the MS
# action='' --> do not run the tool and do not write to the MS
if action != 'apply':
writeflags = False
# Default mode
if mode == '' or mode == 'manualflag':
mode = 'manual'
# Read in the list of commands
# Make a dictionary of the input commands. Select by reason if requested
flagcmd = {}
if mode == 'list':
casalog.post('List mode is active')
doPadding = True
try:
# If tbuff is requested, read and Parse
if tbuff == 0.0 or tbuff == [] or tbuff == None:
doPadding = False
if doPadding:
casalog.post('Will apply time buffer padding')
# inpfile is a file
if isinstance(inpfile, str):
inpfile = [inpfile]
# read in the list and do a simple parsing to apply tbuff
flaglist = fh.readAndParse(inpfile, tbuff)
else:
# inpfile is a file
if isinstance(inpfile, str) and os.path.isfile(inpfile):
flaglist = fh.readFile(inpfile)
nlines = len(flaglist)
casalog.post('Read %s command(s) from file: %s' %
(nlines, inpfile))
# inpfile is a list of files
elif isinstance(inpfile, list) and os.path.isfile(
inpfile[0]):
flaglist = fh.readFiles(inpfile)
# Python list of strings
elif isinstance(inpfile, list):
flaglist = inpfile
else:
raise Exception, 'Unsupported input list of flag commands or input file does not exist'
# Parse and create a dictionary
flagcmd = fh.parseDictionary(flaglist, reason)
# Validate the dictionary.
# IMPORTANT: if any parameter changes its type, the following
# function needs to be updated. The same if any new parameter is
# added or removed from the task
fh.evaluateFlagParameters(flagcmd, orig_locals)
# List of flag commands in dictionary
vrows = flagcmd.keys()
casalog.post('%s' % flagcmd, 'DEBUG1')
except Exception, instance:
casalog.post('%s' % instance, 'ERROR')
raise Exception, 'Error reading the input list. Make sure the syntax used in the list '\
'follows the rules given in the inline help of the task.'
casalog.post('Selected ' + str(vrows.__len__()) +
' commands from combined input list(s) ')
elif mode == 'manual':
agent_pars['autocorr'] = autocorr
casalog.post('Manual mode is active')
def flagdata(vis,
mode,
autocorr, # mode manual parameter
inpfile, # mode list parameters
reason,
tbuff,
spw, # data selection parameters
field,
antenna,
uvrange,
timerange,
correlation,
scan,
intent,
array,
observation,
feed,
clipminmax, # mode clip parameters
datacolumn,
clipoutside,
channelavg,
chanbin,
timeavg,
timebin,
clipzeros,
quackinterval, # mode quack parameters
quackmode,
quackincrement,
tolerance, # mode shadow parameter
addantenna,
lowerlimit, # mode elevation parameters
upperlimit,
ntime, # mode tfcrop
combinescans,
timecutoff,
freqcutoff,
timefit,
freqfit,
maxnpieces,
flagdimension,
usewindowstats,
halfwin,
extendflags,
winsize, # rflag parameters
timedev,
freqdev,
timedevscale,
freqdevscale,
spectralmax,
spectralmin,
extendpols, # mode extend
growtime,
growfreq,
growaround,
flagneartime,
flagnearfreq,
minrel, # mode summary
maxrel,
minabs,
maxabs,
spwchan,
spwcorr,
basecnt,
fieldcnt,
name,
action, # run or not the tool
display,
flagbackup,
savepars, # save the current parameters to FLAG_CMD or to a file
cmdreason, # reason to save to flag cmd
outfile, # output file to save flag commands
overwrite, # overwrite the outfile file
writeflags # HIDDEN parameter
):
#
# Task flagdata
# Flags data from an MS or calibration table based on data selection in various ways
casalog.origin('flagdata')
if (action == 'none' or action=='' or action=='calculate'):
flagbackup = False
# SMC: moved the flagbackup to before initializing the cluster.
# Note that with this change, a flag backup will be created even if
# an error happens that prevents the flagger tool from running.
if (mode != 'summary' and flagbackup):
casalog.post('Backup original flags before applying new flags')
fh.backupFlags(aflocal=None, msfile=vis, prename='flagdata')
# Set flagbackup to False because only the controller
# should create a backup
flagbackup = False
# Initialize the helper class
orig_locals = copy.deepcopy(locals())
FHelper = FlagHelper()
# Check if vis is a MS, MMS or cal table:
# typevis = 1 --> cal table
# typevis = 0 --> MS
# typevis = 2 --> MMS
iscal = False
typevis = fh.isCalTable(vis)
if typevis == 1:
iscal = True
# ***************** Input is MMS -- Parallel Processing ***********************
if FHelper.isMPIEnabled() and typevis == 2 and action != '' and action != 'none':
# Create a temporary input file with .tmp extension.
# Use this file for all the processing from now on.
if (isinstance(inpfile,str) and inpfile != '') or \
(isinstance(inpfile, list) and os.path.isfile(inpfile[0])):
inpfile = FHelper.setupInputFile(inpfile)
if inpfile != None:
orig_locals['inpfile'] = inpfile
if outfile != '':
outfile = os.path.abspath(outfile)
orig_locals['outfile'] = outfile
if isinstance(addantenna, str) and addantenna != '':
addantenna = os.path.abspath(addantenna)
orig_locals['addantenna'] = addantenna
if isinstance(timedev, str) and timedev != '':
timedev = os.path.abspath(timedev)
orig_locals['timedev'] = timedev
if isinstance(freqdev, str) and freqdev != '':
freqdev = os.path.abspath(freqdev)
orig_locals['freqdev'] = freqdev
FHelper.__init__(orig_locals)
# For tests only
# FHelper.bypassParallelProcessing(1)
FHelper.setupCluster('flagdata')
# (CAS-4119): Override summary minabs,maxabs,minrel,maxrel
# so that it is done after consolidating the summaries
# By-pass options to filter summary
filterSummary = False
if ((mode == 'summary') and ((minrel != 0.0) or (maxrel != 1.0) or (minabs != 0) or (maxabs != -1))):
filterSummary = True
myms = mstool()
myms.open(vis)
subMS_list = myms.getreferencedtables()
myms.close()
if (minrel != 0.0):
minreal_dict = create_arg_dict(subMS_list,0.0)
FHelper.override_arg('minrel',minreal_dict)
if (maxrel != 1.0):
maxrel_dict = create_arg_dict(subMS_list,1.0)
FHelper.override_arg('maxrel',maxrel_dict)
if (minabs != 0):
minabs_dict = create_arg_dict(subMS_list,0)
FHelper.override_arg('minabs',minabs_dict)
if (maxabs != -1):
maxabs_dict = create_arg_dict(subMS_list,-1)
FHelper.override_arg('maxabs',maxabs_dict)
# By-pass options to filter summary
if savepars:
myms = mstool()
myms.open(vis)
subMS_list = myms.getreferencedtables()
myms.close()
savepars_dict = create_arg_dict(subMS_list,False)
FHelper.override_arg('savepars',savepars_dict)
# Execute the parallel engines
retVar = FHelper.go()
# In async mode return the job ids
if ParallelTaskHelper.getAsyncMode():
return retVar
else:
# Filter summary at MMS level
if (mode == 'summary'):
if filterSummary:
retVar = filter_summary(retVar,minrel,maxrel,minabs,maxabs)
return retVar
# Save parameters at MMS level
elif savepars:
action = 'none'
else:
return retVar
summary_stats={};
# ***************** Input is a normal MS/cal table ****************
# Create local tools
aflocal = casac.agentflagger()
mslocal = mstool()
try:
# Verify the ntime value
newtime = 0.0
if type(ntime) == float or type(ntime) == int:
if ntime <= 0:
raise Exception, 'Parameter ntime cannot be < = 0'
else:
# units are seconds
newtime = float(ntime)
elif type(ntime) == str:
if ntime == 'scan':
# iteration time step is a scan
newtime = 0.0
else:
# read the units from the string
qtime = qa.quantity(ntime)
if qtime['unit'] == 'min':
# convert to seconds
qtime = qa.convert(qtime, 's')
elif qtime['unit'] == '':
qtime['unit'] = 's'
# check units
if qtime['unit'] == 's':
newtime = qtime['value']
else:
casalog.post('Cannot convert units of ntime. Will use default 0.0s', 'WARN')
casalog.post("New ntime is of type %s and value %s"%(type(newtime),newtime), 'DEBUG')
# Open the MS and attach it to the tool
if ((type(vis) == str) & (os.path.exists(vis))):
aflocal.open(vis, newtime)
else:
raise Exception, 'Visibility data set not found - please verify the name'
# Get the parameters for the mode
agent_pars = {}
# By default, write flags to the MS
writeflags = True
# Only the apply action writes to the MS
# action=apply --> write to the MS
# action=calculate --> do not write to the MS
# action='' --> do not run the tool and do not write to the MS
if action != 'apply':
writeflags = False
# Default mode
if mode == '' or mode == 'manualflag':
mode = 'manual'
# Read in the list of commands
# Make a dictionary of the input commands. Select by reason if requested
flagcmd = {}
if mode == 'list':
casalog.post('List mode is active')
doPadding = True
try:
# If tbuff is requested, read and Parse
if tbuff == 0.0 or tbuff == [] or tbuff == None:
doPadding = False
if doPadding:
casalog.post('Will apply time buffer padding')
# inpfile is a file
if isinstance(inpfile, str):
inpfile = [inpfile]
# read in the list and do a simple parsing to apply tbuff
flaglist = fh.readAndParse(inpfile, tbuff)
else:
# inpfile is a file
if isinstance(inpfile, str) and os.path.isfile(inpfile):
flaglist = fh.readFile(inpfile)
nlines = len(flaglist)
casalog.post('Read %s command(s) from file: %s'%(nlines, inpfile))
# inpfile is a list of files
elif isinstance(inpfile, list) and os.path.isfile(inpfile[0]):
flaglist = fh.readFiles(inpfile)
# Python list of strings
elif isinstance(inpfile, list):
flaglist = inpfile
else:
raise Exception, 'Unsupported input list of flag commands or input file does not exist'
# Parse and create a dictionary
flagcmd = fh.parseDictionary(flaglist, reason)
# Validate the dictionary.
# IMPORTANT: if any parameter changes its type, the following
# function needs to be updated. The same if any new parameter is
# added or removed from the task
fh.evaluateFlagParameters(flagcmd,orig_locals)
# List of flag commands in dictionary
vrows = flagcmd.keys()
casalog.post('%s'%flagcmd,'DEBUG1')
except Exception, instance:
casalog.post('%s'%instance,'ERROR')
raise Exception, 'Error reading the input list. Make sure the syntax used in the list '\
'follows the rules given in the inline help of the task.'
casalog.post('Selected ' + str(vrows.__len__())
+ ' commands from combined input list(s) ')
elif mode == 'manual':
agent_pars['autocorr'] = autocorr
casalog.post('Manual mode is active')