if vrows.__len__() == 0:
raise Exception, 'There are no valid commands in list'
unionpars = {}
if vrows.__len__() == 1:
unionpars = fh.parseSelectionPars(flagcmd[0]['command'])
casalog.post('The selected subset of the MS will be: ')
casalog.post('%s' % unionpars)
aflocal.selectdata(unionpars)
# Parse the parameters for each agent in the list
# Get the returned modified dictionary of flag commands which
# is needed by the rflag agent, when present in a list
modified_flagcmd = fh.parseAgents(aflocal, flagcmd, [], apply,
writeflags, display)
# Do display if requested
if display != '':
agent_pars = {}
casalog.post('Parsing the display parameters')
agent_pars['mode'] = 'display'
# need to create different parameters for both, data and report.
if display == 'both':
agent_pars['datadisplay'] = True
agent_pars['reportdisplay'] = True
elif display == 'data':
agent_pars['datadisplay'] = True
def importevla(
asdm=None,
vis=None,
ocorr_mode=None,
compression=None,
asis=None,
scans=None,
verbose=None,
overwrite=None,
online=None,
tbuff=None,
flagzero=None,
flagpol=None,
shadow=None,
tolerance=None,
addantenna=None,
applyflags=None,
savecmds=None,
outfile=None,
flagbackup=None,
):
""" Convert a Science Data Model (SDM) dataset into a CASA Measurement Set (MS)
....This version is under development and is geared to handling EVLA specific flag and
....system files, and is otherwise equivalent to importasdm.
....
....Keyword arguments:
....asdm -- Name of input SDM file (directory)
........default: none; example: asdm='TOSR0001_sb1308595_1.55294.83601028935'
...."""
# Python script
# Origninator: Steven T. Myers
# Written (3.0.1) STM 2010-03-11 modify importasdm to include flagging from xml
# Vers1.0 (3.0.1) STM 2010-03-16 add tbuff argument
# Vers2.0 (3.0.1) STM 2010-03-29 minor improvements
# Vers3.0 (3.0.2) STM 2010-04-13 add flagzero, doshadow
# Vers4.0 (3.0.2) STM 2010-04-20 add flagpol
# Vers5.0 (3.0.2) STM 2010-05-27 combine flagzero clips
# Vers6.0 (3.1.0) STM 2010-07-01 flagbackup option
# Vers7.0 (3.1.0) STM 2010-08-18 remove corr_mode,wvr_corrected_data,singledish,antenna
# Vers7.1 (3.1.0) STM 2010-10-07 remove time_sampling, srt
# Vers7.1 (3.1.0) STM 2010-10-07 use helper functions, flagger tool, fill FLAG_CMD
# Vers7.2 (3.1.0) STM 2010-10-29 minor modifications to defaults and messages
# Vers8.0 (3.2.0) STM 2010-11-23 tbuff not sub-par of applyflags=T
# Vers8.1 (3.2.0) STM 2010-12-01 prec=9 on timestamps
# Vers8.2 (3.2.0) MKH 2010-12-06 added scan selection
# Vers8.3 (3.2.0) GAM 2011-01-18 added switchedpower option (sw power gain/tsys)
# Vers8.4 (3.2.0) STM 2011-03-24 fix casalog.post line-length bug
# Vers8.5 (3.4.0) STM 2011-12-08 new readflagxml for new Flag.xml format
# Vers8.6 (3.4.0) STM 2011-02-22 full handling of new Flag.xml ant+spw+pol flags
# Vers9.0 (3.4.0) SMC 2012-03-13 ported to use the new flagger tool (agentflagger)
#
# Create local versions of the flagger and ms tools
aflocal = casac.agentflagger()
mslocal = casac.ms()
#
try:
casalog.origin('importevla')
casalog.post('You are using importevla v9.0 SMC Updated 2012-03-13')
viso = ''
casalog.post('corr_mode is forcibly set to all.')
if len(vis) > 0:
viso = vis
else:
viso = asdm + '.ms'
vis = asdm
corr_mode = 'all'
wvr_corrected_data = 'no'
singledish = False
srt = 'all'
time_sampling = 'all'
showversion = True
execute_string = 'asdm2MS --icm "' + corr_mode + '" --isrt "' \
+ srt + '" --its "' + time_sampling + '" --ocm "' \
+ ocorr_mode + '" --wvr-corrected-data "' \
+ wvr_corrected_data + '" --asis "' + asis + '" --scans "' \
+ scans + '" --logfile "' + casalog.logfile() + '"'
if showversion:
casalog.post('asdm2MS --revision --logfile "' + casalog.logfile() +
'"')
os.system('asdm2MS --revision --logfile "' + casalog.logfile() +
'"')
if compression:
execute_string = execute_string + ' --compression'
if verbose:
execute_string = execute_string + ' --verbose'
if not overwrite and os.path.exists(viso):
raise Exception, \
'You have specified and existing ms and have indicated you do not wish to overwrite it'
#
# If viso+".flagversions" then process differently depending on the value of overwrite..
#
dotFlagversion = viso + '.flagversions'
if os.path.exists(dotFlagversion):
if overwrite:
casalog.post("Found '" + dotFlagversion +
"' . It'll be deleted before running the filler.")
os.system('rm -rf %s' % dotFlagversion)
else:
casalog.post("Found '%s' but can't overwrite it." %
dotFlagversion)
raise Exception, "Found '%s' but can't overwrite it." \
% dotFlagversion
execute_string = execute_string + ' ' + asdm + ' ' + viso
casalog.post('Running the asdm2MS standalone invoked as:')
# Print execute_string
casalog.post(execute_string)
# Catch the return status and exit on failure
ret_status = os.system(execute_string)
if ret_status != 0:
casalog.post(
'asdm2MS failed to execute with exit error ' + str(ret_status),
'SEVERE')
raise Exception, 'ASDM conversion error, please check if it is a valid ASDM.'
if compression:
visover = viso
viso = visover.replace('.ms', '.compressed.ms')
if flagbackup:
ok = af.open(viso)
ok = af.saveflagversion('Original',
comment='Original flags on import',
merge='save')
ok = af.done()
print 'Backed up original flag column to ' + viso \
+ '.flagversions'
casalog.post('Backed up original flag column to ' + viso +
'.flagversions')
else:
casalog.post('Warning: will not back up original flag column',
'WARN')
#
# =============================
# Begin EVLA specific code here
# =============================
nflags = 0
# All flag cmds
allflags = {}
if os.access(asdm + '/Flag.xml', os.F_OK):
# Find (and copy) Flag.xml
print ' Found Flag.xml in SDM, copying to MS'
casalog.post('Found Flag.xml in SDM, copying to MS')
os.system('cp -rf ' + asdm + '/Flag.xml ' + viso + '/')
# Find (and copy) Antenna.xml
if os.access(asdm + '/Antenna.xml', os.F_OK):
print ' Found Antenna.xml in SDM, copying to MS'
casalog.post('Found Antenna.xml in SDM, copying to MS')
os.system('cp -rf ' + asdm + '/Antenna.xml ' + viso + '/')
else:
raise Exception, 'Failed to find Antenna.xml in SDM'
# Find (and copy) SpectralWindow.xml
if os.access(asdm + '/SpectralWindow.xml', os.F_OK):
print ' Found SpectralWindow.xml in SDM, copying to MS'
casalog.post('Found SpectralWindow.xml in SDM, copying to MS')
os.system('cp -rf ' + asdm + '/SpectralWindow.xml ' + viso +
'/')
else:
raise Exception, \
'Failed to find SpectralWindow.xml in SDM'
#
# Parse Flag.xml into flag dictionary
#
if online:
# flago = fh.readXML(asdm, tbuff)
flago = fh.parseXML(asdm, tbuff)
onlinekeys = flago.keys()
nkeys = onlinekeys.__len__()
nflags += nkeys
allflags = flago.copy()
casalog.post('Created %s commands for online flags' %
str(nflags))
else:
if online:
casalog.post('ERROR: No Flag.xml in SDM', 'SEVERE')
else:
casalog.post('WARNING: No Flag.xml in SDM', 'WARN')
if flagzero or shadow:
# Get overall MS time range for later use (if needed)
(ms_startmjds, ms_endmjds, ms_starttime, ms_endtime) = \
getmsmjds(viso)
# Now add zero and shadow flags
if flagzero:
flagz = {}
# clip zero data
# NOTE: currently hard-wired to RL basis
# assemble into flagging commands and add to myflagd
flagz['time'] = 0.5 * (ms_startmjds + ms_endmjds)
flagz['interval'] = ms_endmjds - ms_startmjds
flagz['level'] = 0
flagz['severity'] = 0
flagz['type'] = 'FLAG'
flagz['applied'] = False
flagz['antenna'] = ''
flagz['mode'] = 'clip'
# Flag cross-hands too
if flagpol:
flagz['reason'] = 'CLIP_ZERO_ALL'
# flagz['command'] = \
# "mode='clip' clipzeros=True correlation='ABS_ALL'"
command = {}
command['mode'] = 'clip'
command['clipzeros'] = True
command['correlation'] = 'ABS_ALL'
flagz['command'] = command
flagz['id'] = 'ZERO_ALL'
allflags[nflags] = flagz.copy()
nflags += 1
nflagz = 1
else:
flagz['reason'] = 'CLIP_ZERO_RR'
# flagz['command'] = \
# "mode='clip' clipzeros=True correlation='ABS_RR'"
command = {}
command['mode'] = 'clip'
command['clipzeros'] = True
command['correlation'] = 'ABS_RR'
flagz['command'] = command
flagz['id'] = 'ZERO_RR'
allflags[nflags] = flagz.copy()
nflags += 1
flagz['reason'] = 'CLIP_ZERO_LL'
# flagz['command'] = \
# "mode='clip' clipzeros=True correlation='ABS_LL'"
command = {}
command['mode'] = 'clip'
command['clipzeros'] = True
command['correlation'] = 'ABS_LL'
flagz['command'] = command
flagz['id'] = 'ZERO_LL'
allflags[nflags] = flagz.copy()
nflags += 1
nflagz = 2
casalog.post('Created %s command(s) to clip zeros' % str(nflagz))
if shadow:
flagh = {}
command = {}
# flag shadowed data
flagh['time'] = 0.5 * (ms_startmjds + ms_endmjds)
flagh['interval'] = ms_endmjds - ms_startmjds
flagh['level'] = 0
flagh['severity'] = 0
flagh['type'] = 'FLAG'
flagh['applied'] = False
flagh['antenna'] = ''
flagh['mode'] = 'shadow'
flagh['reason'] = 'SHADOW'
# scmd = 'mode=shadow tolerance=' + str(tolerance)
# scmd = "mode='shadow' tolerance=" + str(tolerance)
command['mode'] = 'shadow'
command['tolerance'] = tolerance
if type(addantenna) == str:
if addantenna != '':
# it's a filename, create a dictionary
antdict = fh.readAntennaList(addantenna)
# scmd = scmd +' addantenna='+str(antdict)
command['addantenna'] = antdict
elif type(addantenna) == dict:
if addantenna != {}:
# scmd = scmd +' addantenna='+str(addantenna)
command['addantenna'] = addantenna
# flagh['command'] = scmd
flagh['command'] = command
flagh['id'] = 'SHADOW'
allflags[nflags] = flagh.copy()
nflags += 1
casalog.post('Created 1 command to flag shadowed data')
# List of rows to save
allkeys = allflags.keys()
# Apply the flags
if applyflags:
if nflags > 0:
# Open the MS and attach the tool
aflocal.open(viso)
# Select the data
aflocal.selectdata()
# Setup the agent's parameters
fh.parseAgents(aflocal, allflags, [], True, True, '')
# Initialize the agents
aflocal.init()
# Run the tool
stats = aflocal.run(True, True)
casalog.post('Applied %s flag commands to data' % str(nflags))
# Destroy the tool
aflocal.done()
# Save the flags to FLAG_CMD and update the APPLIED column
# fh.writeFlagCmd(viso, allflags, allkeys, True, '', '')
fh.writeFlagCommands(viso, allflags, True, '', '', True)
else:
casalog.post('There are no flags to apply')
else:
casalog.post(
'Will not apply flags (applyflags=False), use flagcmd to apply'
)
if nflags > 0:
# fh.writeFlagCmd(viso, allflags, allkeys, False, '', '')
fh.writeFlagCommands(viso, allflags, False, '', '', True)
# Save the flag commads to an ASCII file
if savecmds:
if nflags > 0:
# Save the cmds to a file
if outfile == '':
# Save to standard filename
outfile = viso.replace('.ms', '_cmd.txt')
# fh.writeFlagCmd(viso, allflags, allkeys, False, '', outfile)
fh.writeFlagCommands(viso, allflags, False, '', outfile, True)
casalog.post('Saved %s flag commands to %s' %
(nflags, outfile))
else:
casalog.post('There are no flag commands to save')
except Exception, instance:
casalog.post('%s' % instance, 'ERROR')
if vrows.__len__() == 0:
raise Exception, 'There are no valid commands in list'
unionpars = {}
if vrows.__len__() == 1:
unionpars = fh.parseSelectionPars(flagcmd[0]['command'])
casalog.post('The selected subset of the MS will be: ');
casalog.post('%s'%unionpars);
aflocal.selectdata(unionpars);
# Parse the parameters for each agent in the list
# Get the returned modified dictionary of flag commands which
# is needed by the rflag agent, when present in a list
modified_flagcmd = fh.parseAgents(aflocal, flagcmd, [], apply, writeflags, display)
# Do display if requested
if display != '':
agent_pars = {}
casalog.post('Parsing the display parameters')
agent_pars['mode'] = 'display'
# need to create different parameters for both, data and report.
if display == 'both':
agent_pars['datadisplay'] = True
agent_pars['reportdisplay'] = True
elif display == 'data':
agent_pars['datadisplay'] = True
if process_flags:
flagcmds = fh.parseXML(asdm, float(tbuff))
onlinekeys = flagcmds.keys()
nflags = onlinekeys.__len__()
# Apply flags to the MS
if nflags > 0:
idx = 0
for myviso in vistoproc:
if applyflags:
# Open the MS and attach it to the tool
aflocal.open(myviso)
# Select the data
aflocal.selectdata()
# Setup the agent's parameters
fh.parseAgents(aflocal, flagcmds, [], True, True, '')
# Initialize the agents
aflocal.init()
# Run the tool
aflocal.run(True, True)
casalog.post('Applied %s flag commands to %s'%(nflags,myviso))
# Destroy the tool and de-attach the MS
aflocal.done()
# Save to FLAG_CMD table. APPLIED is set to True.
fh.writeFlagCommands(myviso, flagcmds, True, '', '', True)
else:
casalog.post('Will not apply flags to %s (apply_flags=False), use flagcmd to apply'%myviso)
# Write to FLAG_CMD, APPLIED is set to False
fh.writeFlagCommands(myviso, flagcmds, False, '', '', True)
def importevla(
asdm=None,
vis=None,
ocorr_mode=None,
compression=None,
asis=None,
scans=None,
verbose=None,
overwrite=None,
online=None,
tbuff=None,
flagzero=None,
flagpol=None,
shadow=None,
tolerance=None,
addantenna=None,
applyflags=None,
savecmds=None,
outfile=None,
flagbackup=None,
):
""" Convert a Science Data Model (SDM) dataset into a CASA Measurement Set (MS)
....This version is under development and is geared to handling EVLA specific flag and
....system files, and is otherwise equivalent to importasdm.
....
....Keyword arguments:
....asdm -- Name of input SDM file (directory)
........default: none; example: asdm='TOSR0001_sb1308595_1.55294.83601028935'
...."""
# Python script
# Origninator: Steven T. Myers
# Written (3.0.1) STM 2010-03-11 modify importasdm to include flagging from xml
# Vers1.0 (3.0.1) STM 2010-03-16 add tbuff argument
# Vers2.0 (3.0.1) STM 2010-03-29 minor improvements
# Vers3.0 (3.0.2) STM 2010-04-13 add flagzero, doshadow
# Vers4.0 (3.0.2) STM 2010-04-20 add flagpol
# Vers5.0 (3.0.2) STM 2010-05-27 combine flagzero clips
# Vers6.0 (3.1.0) STM 2010-07-01 flagbackup option
# Vers7.0 (3.1.0) STM 2010-08-18 remove corr_mode,wvr_corrected_data,singledish,antenna
# Vers7.1 (3.1.0) STM 2010-10-07 remove time_sampling, srt
# Vers7.1 (3.1.0) STM 2010-10-07 use helper functions, flagger tool, fill FLAG_CMD
# Vers7.2 (3.1.0) STM 2010-10-29 minor modifications to defaults and messages
# Vers8.0 (3.2.0) STM 2010-11-23 tbuff not sub-par of applyflags=T
# Vers8.1 (3.2.0) STM 2010-12-01 prec=9 on timestamps
# Vers8.2 (3.2.0) MKH 2010-12-06 added scan selection
# Vers8.3 (3.2.0) GAM 2011-01-18 added switchedpower option (sw power gain/tsys)
# Vers8.4 (3.2.0) STM 2011-03-24 fix casalog.post line-length bug
# Vers8.5 (3.4.0) STM 2011-12-08 new readflagxml for new Flag.xml format
# Vers8.6 (3.4.0) STM 2011-02-22 full handling of new Flag.xml ant+spw+pol flags
# Vers9.0 (3.4.0) SMC 2012-03-13 ported to use the new flagger tool (agentflagger)
#
# Create local versions of the flagger and ms tools
aflocal = casac.agentflagger()
mslocal = casac.ms()
#
try:
casalog.origin('importevla')
casalog.post('You are using importevla v9.0 SMC Updated 2012-03-13'
)
viso = ''
casalog.post('corr_mode is forcibly set to all.')
if len(vis) > 0:
viso = vis
else:
viso = asdm + '.ms'
vis = asdm
corr_mode = 'all'
wvr_corrected_data = 'no'
singledish = False
srt = 'all'
time_sampling = 'all'
showversion = True
execute_string = 'asdm2MS --icm "' + corr_mode + '" --isrt "' \
+ srt + '" --its "' + time_sampling + '" --ocm "' \
+ ocorr_mode + '" --wvr-corrected-data "' \
+ wvr_corrected_data + '" --asis "' + asis + '" --scans "' \
+ scans + '" --logfile "' + casalog.logfile() + '"'
if showversion:
casalog.post('asdm2MS --revision --logfile "'
+ casalog.logfile() + '"')
os.system('asdm2MS --revision --logfile "'
+ casalog.logfile() + '"')
if compression:
execute_string = execute_string + ' --compression'
if verbose:
execute_string = execute_string + ' --verbose'
if not overwrite and os.path.exists(viso):
raise Exception, \
'You have specified and existing ms and have indicated you do not wish to overwrite it'
#
# If viso+".flagversions" then process differently depending on the value of overwrite..
#
dotFlagversion = viso + '.flagversions'
if os.path.exists(dotFlagversion):
if overwrite:
casalog.post("Found '" + dotFlagversion
+ "' . It'll be deleted before running the filler."
)
os.system('rm -rf %s' % dotFlagversion)
else:
casalog.post("Found '%s' but can't overwrite it."
% dotFlagversion)
raise Exception, "Found '%s' but can't overwrite it." \
% dotFlagversion
execute_string = execute_string + ' ' + asdm + ' ' + viso
casalog.post('Running the asdm2MS standalone invoked as:')
# Print execute_string
casalog.post(execute_string)
# Catch the return status and exit on failure
ret_status = os.system(execute_string)
if ret_status != 0:
casalog.post('asdm2MS failed to execute with exit error '+str(ret_status), 'SEVERE')
raise Exception, 'ASDM conversion error, please check if it is a valid ASDM.'
if compression:
visover = viso
viso = visover.replace('.ms','.compressed.ms')
if flagbackup:
ok = af.open(viso)
ok = af.saveflagversion('Original',
comment='Original flags on import', merge='save')
ok = af.done()
print 'Backed up original flag column to ' + viso \
+ '.flagversions'
casalog.post('Backed up original flag column to ' + viso
+ '.flagversions')
else:
casalog.post('Warning: will not back up original flag column'
, 'WARN')
#
# =============================
# Begin EVLA specific code here
# =============================
nflags = 0
# All flag cmds
allflags = {}
if os.access(asdm + '/Flag.xml', os.F_OK):
# Find (and copy) Flag.xml
print ' Found Flag.xml in SDM, copying to MS'
casalog.post('Found Flag.xml in SDM, copying to MS')
os.system('cp -rf ' + asdm + '/Flag.xml ' + viso + '/')
# Find (and copy) Antenna.xml
if os.access(asdm + '/Antenna.xml', os.F_OK):
print ' Found Antenna.xml in SDM, copying to MS'
casalog.post('Found Antenna.xml in SDM, copying to MS')
os.system('cp -rf ' + asdm + '/Antenna.xml ' + viso
+ '/')
else:
raise Exception, 'Failed to find Antenna.xml in SDM'
# Find (and copy) SpectralWindow.xml
if os.access(asdm + '/SpectralWindow.xml', os.F_OK):
print ' Found SpectralWindow.xml in SDM, copying to MS'
casalog.post('Found SpectralWindow.xml in SDM, copying to MS'
)
os.system('cp -rf ' + asdm + '/SpectralWindow.xml '
+ viso + '/')
else:
raise Exception, \
'Failed to find SpectralWindow.xml in SDM'
#
# Parse Flag.xml into flag dictionary
#
if online:
# flago = fh.readXML(asdm, tbuff)
flago = fh.parseXML(asdm, tbuff)
onlinekeys = flago.keys()
nkeys = onlinekeys.__len__()
nflags += nkeys
allflags = flago.copy()
casalog.post('Created %s commands for online flags'%str(nflags))
else:
if online:
casalog.post('ERROR: No Flag.xml in SDM', 'SEVERE')
else:
casalog.post('WARNING: No Flag.xml in SDM', 'WARN')
if flagzero or shadow:
# Get overall MS time range for later use (if needed)
(ms_startmjds, ms_endmjds, ms_starttime, ms_endtime) = \
getmsmjds(viso)
# Now add zero and shadow flags
if flagzero:
flagz = {}
# clip zero data
# NOTE: currently hard-wired to RL basis
# assemble into flagging commands and add to myflagd
flagz['time'] = 0.5 * (ms_startmjds + ms_endmjds)
flagz['interval'] = ms_endmjds - ms_startmjds
flagz['level'] = 0
flagz['severity'] = 0
flagz['type'] = 'FLAG'
flagz['applied'] = False
flagz['antenna'] = ''
flagz['mode'] = 'clip'
# Flag cross-hands too
if flagpol:
flagz['reason'] = 'CLIP_ZERO_ALL'
# flagz['command'] = \
# "mode='clip' clipzeros=True correlation='ABS_ALL'"
command = {}
command['mode'] = 'clip'
command['clipzeros'] = True
command['correlation'] = 'ABS_ALL'
flagz['command'] = command
flagz['id'] = 'ZERO_ALL'
allflags[nflags] = flagz.copy()
nflags += 1
nflagz = 1
else:
flagz['reason'] = 'CLIP_ZERO_RR'
# flagz['command'] = \
# "mode='clip' clipzeros=True correlation='ABS_RR'"
command = {}
command['mode'] = 'clip'
command['clipzeros'] = True
command['correlation'] = 'ABS_RR'
flagz['command'] = command
flagz['id'] = 'ZERO_RR'
allflags[nflags] = flagz.copy()
nflags += 1
flagz['reason'] = 'CLIP_ZERO_LL'
# flagz['command'] = \
# "mode='clip' clipzeros=True correlation='ABS_LL'"
command = {}
command['mode'] = 'clip'
command['clipzeros'] = True
command['correlation'] = 'ABS_LL'
flagz['command'] = command
flagz['id'] = 'ZERO_LL'
allflags[nflags] = flagz.copy()
nflags += 1
nflagz = 2
casalog.post('Created %s command(s) to clip zeros'%str(nflagz))
if shadow:
flagh = {}
command = {}
# flag shadowed data
flagh['time'] = 0.5 * (ms_startmjds + ms_endmjds)
flagh['interval'] = ms_endmjds - ms_startmjds
flagh['level'] = 0
flagh['severity'] = 0
flagh['type'] = 'FLAG'
flagh['applied'] = False
flagh['antenna'] = ''
flagh['mode'] = 'shadow'
flagh['reason'] = 'SHADOW'
# scmd = 'mode=shadow tolerance=' + str(tolerance)
# scmd = "mode='shadow' tolerance=" + str(tolerance)
command['mode'] = 'shadow'
command['tolerance'] = tolerance
if type(addantenna) == str:
if addantenna != '':
# it's a filename, create a dictionary
antdict = fh.readAntennaList(addantenna)
# scmd = scmd +' addantenna='+str(antdict)
command['addantenna'] = antdict
elif type(addantenna) == dict:
if addantenna != {}:
# scmd = scmd +' addantenna='+str(addantenna)
command['addantenna'] = addantenna
# flagh['command'] = scmd
flagh['command'] = command
flagh['id'] = 'SHADOW'
allflags[nflags] = flagh.copy()
nflags += 1
casalog.post('Created 1 command to flag shadowed data')
# List of rows to save
allkeys = allflags.keys()
# Apply the flags
if applyflags:
if nflags > 0:
# Open the MS and attach the tool
aflocal.open(viso)
# Select the data
aflocal.selectdata()
# Setup the agent's parameters
fh.parseAgents(aflocal, allflags, [], True, True, '')
# Initialize the agents
aflocal.init()
# Run the tool
stats = aflocal.run(True, True)
casalog.post('Applied %s flag commands to data'%str(nflags))
# Destroy the tool
aflocal.done()
# Save the flags to FLAG_CMD and update the APPLIED column
# fh.writeFlagCmd(viso, allflags, allkeys, True, '', '')
fh.writeFlagCommands(viso,allflags,True,'','', True)
else:
casalog.post('There are no flags to apply')
else :
casalog.post('Will not apply flags (applyflags=False), use flagcmd to apply')
if nflags > 0:
# fh.writeFlagCmd(viso, allflags, allkeys, False, '', '')
fh.writeFlagCommands(viso,allflags,False,'','', True)
# Save the flag commads to an ASCII file
if savecmds:
if nflags > 0:
# Save the cmds to a file
if outfile == '':
# Save to standard filename
outfile = viso.replace('.ms','_cmd.txt')
# fh.writeFlagCmd(viso, allflags, allkeys, False, '', outfile)
fh.writeFlagCommands(viso,allflags,False,'',outfile, True)
casalog.post('Saved %s flag commands to %s'%(nflags,outfile))
else:
casalog.post('There are no flag commands to save')
except Exception, instance:
casalog.post('%s' % instance, 'ERROR')
if process_flags:
flagcmds = fh.parseXML(asdm, float(tbuff))
onlinekeys = flagcmds.keys()
nflags = onlinekeys.__len__()
# Apply flags to the MS
if nflags > 0:
idx = 0
for myviso in vistoproc:
if applyflags:
# Open the MS and attach it to the tool
aflocal.open(myviso)
# Select the data
aflocal.selectdata()
# Setup the agent's parameters
fh.parseAgents(aflocal, flagcmds, [], True, True,
'')
# Initialize the agents
aflocal.init()
# Run the tool
aflocal.run(True, True)
casalog.post('Applied %s flag commands to %s' %
(nflags, myviso))
# Destroy the tool and de-attach the MS
aflocal.done()
# Save to FLAG_CMD table. APPLIED is set to True.
fh.writeFlagCommands(myviso, flagcmds, True, '',
'', True)
else:
casalog.post(
'Will not apply flags to %s (apply_flags=False), use flagcmd to apply'
% myviso)
def importasdm(asdm=None,
vis=None,
createmms=None,
separationaxis=None,
numsubms=None,
corr_mode=None,
srt=None,
time_sampling=None,
ocorr_mode=None,
compression=None,
lazy=None,
asis=None,
wvr_corrected_data=None,
scans=None,
ignore_time=None,
process_syspower=None,
process_caldevice=None,
process_pointing=None,
process_flags=None,
tbuff=None,
applyflags=None,
savecmds=None,
outfile=None,
flagbackup=None,
verbose=None,
overwrite=None,
showversion=None,
useversion=None,
bdfflags=None,
with_pointing_correction=None,
remove_ref_undef=None,
convert_ephem2geo=None,
polyephem_tabtimestep=None):
"""Convert an ALMA Science Data Model observation into a CASA visibility file (MS) or single-dish data format (Scantable).
The conversion of the ALMA SDM archive format into a measurement set. This version
is under development and is geared to handling many spectral windows of different
shapes.
Keyword arguments:
asdm -- Name of input ASDM file (directory)
default: none; example: asdm='ExecBlock3'
vis -- Root ms or scantable name, note a prefix (.ms or .asap) is NOT appended to this name
default: none
createmms -- Create a Multi-MS
default: False
corr_mode -- correlation mode to be considered on input. Could
be one or more of the following, ao, co, ac, or all
default: all
srt -- spectral resolution type. Could be one or more of
the following, fr, ca, bw, or all
default: all
time_sampling -- specifies the time sampling, INTEGRATION and/or
SUBINTEGRATION. could be one or more of the following
i, si, or all.
default: all
ocorr_mode -- output data for correlation mode AUTO_ONLY
(ao) or CROSS_ONLY (co) or CROSS_AND_AUTO (ca)
default: ca
compression -- produces comrpressed columns in the resulting measurement set.
default: False
lazy -- Make the MS DATA column read the ASDM Binary data directly
(faster import, smaller MS)
default: False
asis -- creates verbatim copies of the ASDM tables in
the output measurement set. The value given to
this option must be a list of table names separated
by space characters; the wildcard character '*' is
allowed in table names.
wvr_corrected_data -- specifies which values are considered in the
ASDM binary data to fill the DATA column in
the MAIN table of the MS. Expected values for
this option are 'no' for the uncorrected data
(this is the default), 'yes' for the corrected
data and 'both' for corrected and uncorrected
data. In the latter case, two measurement sets
are created, one containing the uncorrected
data and the other one, whose name is suffixed
by '-wvr-corrected', containing the corrected
data.
scans -- processes only the scans specified in the option's value. This value is a semicolon
separated list of scan specifications. A scan specification consists in an exec bock index
followed by the character ':' followed by a comma separated list of scan indexes or scan
index ranges. A scan index is relative to the exec block it belongs to. Scan indexes are
1-based while exec blocks's are 0-based. "0:1" or "2:2~6" or "0:1,1:2~6,8;2:,3:24~30" "1,2"
are valid values for the option. "3:" alone will be interpreted as 'all the scans of the
exec block#3'. An scan index or a scan index range not preceded by an exec block index will
be interpreted as 'all the scans with such indexes in all the exec blocks'. By default
all the scans are considered.
ignore_time -- All the rows of the tables Feed, History, Pointing, Source, SysCal, CalDevice, SysPower,
and Weather are processed independently of the time range of the selected exec block / scan.
process_syspower -- The SysPower table is processed if and only if this parameter is set to True.
default: True
process_caldevice -- The CalDevice table is processed if and only if this parameter is set to True.
default: True
process_pointing -- The Pointing table is processed if and only if this parameter is set to True.
If the parameter is set to False the resulting MS will have an empty POINTING table.
default: True
process_flags -- Process the online flags and save them to the FLAG_CMD sub-table.
default: True
>>> process_flags expandable parameter
tbuff -- Time padding buffer (in seconds).
default: 0.0
applyflags -- Apply the online flags to the MS.
default: False
savecmds -- Save the online flags to an ASCII file.
default: False
outfile -- Filename to save the online flags.
default: ''
flagbackup -- Backup the FLAG column in the .flagversions.
default: True
verbose -- produce log output as asdm2MS is being run.
overwrite -- Over write an existing MS.
showversion -- report the version of the asdm2MS being used.
useversion -- Selects the version of asdm2MS to be used (presently only \'v3\' is available).
default: v3
bdfflags -- Set the MS FLAG column according to the ASDM _binary_ flags
default: false
with_pointing_correction -- add (ASDM::Pointing::encoder - ASDM::Pointing::pointingDirection)
to the value to be written in MS::Pointing::direction
default: false
remove_ref_undef -- if set to True then apply fixspwbackport on the resulting MSes.
convert_ephem2geo -- if True, convert any attached ephemerides to the GEO reference frame
polyephem_tabtimestep -- Timestep (days) for the tabulation of polynomial ephemerides. A value <= 0 disables tabulation.
Presently, VLA data can contain polynomial ephemerides. ALMA data uses tabulated values.
default: 0.
"""
# Python script
# make agentflagger tool local
aflocal = casac.agentflagger()
# make table tool local
tblocal = casac.table()
try:
casalog.origin('importasdm')
viso = ''
visoc = '' # for the wvr corrected version, if needed
if len(vis) > 0:
viso = vis
tmps = vis.rstrip('.ms')
if tmps == vis:
visoc = vis + '-wvr-corrected'
else:
visoc = tmps + '-wvr-corrected.ms'
else:
viso = asdm.rstrip("/") + '.ms'
visoc = asdm.rstrip("/") + '-wvr-corrected.ms'
vis = asdm.rstrip("/")
useversion = 'v3'
theexecutable = 'asdm2MS'
execute_string = theexecutable + ' --icm "' + corr_mode \
+ '" --isrt "' + srt + '" --its "' + time_sampling \
+ '" --ocm "' + ocorr_mode + '" --wvr-corrected-data "' \
+ wvr_corrected_data + '" --asis "' + asis \
+ '" --logfile "' + casalog.logfile() + '"'
if len(scans) > 0:
execute_string = execute_string + ' --scans ' + scans
if ignore_time:
execute_string = execute_string + ' --ignore-time'
if useversion == 'v3':
if not process_syspower:
execute_string = execute_string + ' --no-syspower'
if not process_caldevice:
execute_string = execute_string + ' --no-caldevice'
if not process_pointing:
execute_string = execute_string + ' --no-pointing'
if compression:
execute_string = execute_string + ' --compression'
elif lazy:
execute_string = execute_string + ' --lazy'
if verbose:
execute_string = execute_string + ' --verbose'
# if not overwrite and os.path.exists(viso):
# raise Exception, \
# 'You have specified an existing MS and have indicated you do not wish to overwrite it'
# Compression
if compression:
# viso = viso + '.compressed'
viso = viso.rstrip('.ms') + '.compressed.ms'
visoc = visoc.rstrip('.ms') + '.compressed.ms'
vistoproc = [] # the output MSs to post-process
if wvr_corrected_data == 'no' or wvr_corrected_data == 'both':
vistoproc.append(viso)
if (wvr_corrected_data == 'yes' or wvr_corrected_data == 'both'):
vistoproc.append(visoc)
for ff in vistoproc:
if not overwrite and os.path.exists(ff):
raise Exception, \
'You have specified an existing MS and have indicated you do not wish to overwrite it: %s'%ff
# If viso+".flagversions" then process differently depending on the value of overwrite..
#
if flagbackup:
for myviso in vistoproc:
dotFlagversion = myviso + '.flagversions'
if os.path.exists(dotFlagversion):
if overwrite:
casalog.post(
"Found '" + dotFlagversion +
"' . It'll be deleted before running the filler.")
os.system('rm -rf %s' % dotFlagversion)
else:
casalog.post("Found '%s' but can't overwrite it." %
dotFlagversion)
raise Exception, "Found '%s' but can't overwrite it." \
% dotFlagversion
# Make outfile always a list
if isinstance(outfile, str):
if outfile == '':
outfile = []
else:
noutfile = [outfile]
outfile = noutfile
if savecmds:
if len(outfile) == 0:
# Create default names for the online flags
for myviso in vistoproc:
outfile.append(myviso.replace('.ms', '_cmd.txt'))
elif len(outfile) != len(vistoproc):
casalog.post(
'List of outfile names does not match list of MSs', 'WARN')
casalog.post('Will save online flags to temporary filenames',
'WARN')
outfile = []
for myviso in vistoproc:
online_file = myviso.replace('.ms', '_TEMP_cmd.txt')
outfile.append(online_file)
if not overwrite:
for of in outfile:
if os.path.exists(of):
raise Exception, "Cannot overwrite online flags file '%s'; overwrite is set to False." % of
execute_string = execute_string + ' ' + asdm + ' ' + viso
if showversion:
casalog.post(
"You set option \'showversion\' to True. Will just output the version information and then terminate.",
'WARN')
execute_string = theexecutable + ' --revision'
if with_pointing_correction:
execute_string = execute_string + ' --with-pointing-correction'
if (polyephem_tabtimestep !=
None) and (type(polyephem_tabtimestep) == int
or type(polyephem_tabtimestep) == float):
if polyephem_tabtimestep > 0:
casalog.post(
'Will tabulate all attached polynomial ephemerides with a time step of '
+ str(polyephem_tabtimestep) + ' days.')
if polyephem_tabtimestep > 1.:
casalog.post(
'A tabulation timestep of <= 1 days is recommended.',
'WARN')
execute_string = execute_string + ' --polyephem-tabtimestep ' + str(
polyephem_tabtimestep)
casalog.post('Running ' + theexecutable + ' standalone invoked as:')
# print execute_string
casalog.post(execute_string)
exitcode = os.system(execute_string)
if exitcode != 0:
if not showversion:
casalog.post(
theexecutable + ' terminated with exit code ' +
str(exitcode), 'SEVERE')
raise Exception, \
'ASDM conversion error. Please check if it is a valid ASDM and that data/alma/asdm is up to date.'
if showversion:
return
#
# Possibly remove the name of the measurement set expected to contain the corrected data from the list of of produced measurement
# sets if it appears the filler did not find any corrected data.
#
if not os.path.exists(visoc):
vistoproc = [myviso for myviso in vistoproc if myviso != visoc]
#
# Do we apply fixspwbackport
if remove_ref_undef:
casalog.post(
'remove_ref_undef=True: fixspwbackport will be applied ...')
for myviso in vistoproc:
cmd = 'fixspwbackport ' + myviso
casalog.post('Running fixspwbackport standalone invoked as:')
casalog.post(cmd)
cmdexitcode = os.system(cmd)
if cmdexitcode != 0:
casalog.post(
cmd + ' terminated with exit code ' + str(cmdexitcode),
'SEVERE')
raise Exception, 'fixspwbackport error.'
# Binary Flag processing
if bdfflags:
casalog.post(
'Parameter bdfflags==True: flags from the ASDM binary data will be used to set the MS flags ...'
)
bdffexecutable = 'bdflags2MS '
bdffexecstring_base = bdffexecutable + ' -f ALL' + ' --ocm "' + ocorr_mode \
+ '" --logfile "' + casalog.logfile() + '"'
if len(scans) > 0:
bdffexecstring_base = bdffexecstring_base + ' --scans ' + scans
if lazy and not compression:
bdffexecstring_base = bdffexecstring_base + ' --lazy=true'
for myviso in vistoproc:
if myviso.find("wvr-corrected") != -1:
options = " --wvr-corrected=True "
else:
options = " "
bdffexecstring = bdffexecstring_base + options + asdm + ' ' + myviso
casalog.post('Running ' + bdffexecutable +
' standalone invoked as:')
casalog.post(bdffexecstring)
bdffexitcode = os.system(bdffexecstring)
if bdffexitcode != 0:
casalog.post(
bdffexecutable + ' terminated with exit code ' +
str(bdffexitcode), 'SEVERE')
raise Exception, \
'ASDM binary flags conversion error. Please check if it is a valid ASDM and that data/alma/asdm is up to date.'
theephemfields = ce.findattachedephemfields(myviso, field='*')
if len(theephemfields) > 0:
# until asdm2MS does this internally: recalc the UVW coordinates for ephem fields
imt = imtool()
imt.open(myviso, usescratch=False)
imt.calcuvw(theephemfields, refcode='J2000', reuse=False)
imt.close()
if convert_ephem2geo:
for myviso in vistoproc:
ce.convert2geo(myviso,
'*') # convert any attached ephemerides to GEO
if len(theephemfields) > 0:
# also set the direction column in the SOURCE table
tblocal.open(myviso + '/FIELD', nomodify=False)
sourceids = tblocal.getcol('SOURCE_ID')
ftimes = tblocal.getcol('TIME')
ftimekw = tblocal.getcolkeywords('TIME')
tmpa = tblocal.getcol('PHASE_DIR')
origphasedir = tmpa
affectedsids = []
thesamplefields = []
for fld in theephemfields: # determine all source ids used by the ephem fields
if not (sourceids[fld]
in affectedsids): # this source id wasn't handled yet
affectedsids.append(sourceids[fld])
thesamplefields.append(fld)
# need to temporarily change the offset (not all mosaics have an element at (0,0))
tmpa[0][0][fld] = 0.
tmpa[1][0][fld] = 0.
#endif
#endfor
tblocal.putcol('PHASE_DIR', tmpa)
tblocal.close()
tblocal.open(myviso + '/SOURCE')
sourceposref = tblocal.getcolkeywords(
'DIRECTION')['MEASINFO']['Ref']
tblocal.close()
directions = []
msmdlocal = casac.msmetadata()
msmdlocal.open(myviso)
for fld in thesamplefields:
thedirmeas = msmdlocal.phasecenter(fld)
if thedirmeas['refer'] != sourceposref:
casalog.post(
'Ephemeris is in ' + thedirmeas['refer'] +
' instead of ' + sourceposref +
' frame.\nEntry in SOURCE table will be converted to '
+ sourceposref, 'WARN')
melocal = metool()
melocal.doframe(thedirmeas)
thedirmeas = melocal.measure(thedirmeas, sourceposref)
directions.append(
[thedirmeas['m0']['value'], thedirmeas['m1']['value']])
thetime = me.epoch(v0=str(ftimes[fld]) + 's',
rf=ftimekw['MEASINFO']['Ref'])
casalog.post("Will set SOURCE direction for SOURCE_ID " +
str(sourceids[fld]) +
" to ephemeris phase center for time " +
str(thetime['m0']['value']) + " " +
thetime['m0']['unit'] + " " + thetime['refer'])
#endfor
msmdlocal.close()
# restore original PHASE_DIR
tblocal.open(myviso + '/FIELD', nomodify=False)
tblocal.putcol('PHASE_DIR', origphasedir)
tblocal.close()
# write source directions
tblocal.open(myviso + '/SOURCE', nomodify=False)
ssourceids = tblocal.getcol('SOURCE_ID')
sdirs = tblocal.getcol('DIRECTION')
for row in xrange(0, len(ssourceids)):
for i in xrange(0, len(affectedsids)):
if ssourceids[row] == affectedsids[i]:
sdirs[0][row] = directions[i][0]
sdirs[1][row] = directions[i][1]
break
#endfor
#endfor
tblocal.putcol('DIRECTION',
sdirs) # write back corrected directions
tblocal.close()
#end if
##############################################################################################3
# CAS-7369 - Create an output Multi-MS (MMS)
if createmms:
# Get the default parameters of partition
from tasks import partition
fpars = partition.parameters
for mypar in fpars.keys():
fpars[mypar] = partition.itsdefault(mypar)
# Call the cluster for each MS
for myviso in vistoproc:
casalog.origin('importasdm')
# Move original MS to tempdir
tempname = myviso + '.temp.ms'
outputmms = myviso
shutil.move(myviso, tempname)
# Get the proper column
datacolumn = 'DATA'
dcols = ['DATA', 'FLOAT_DATA']
for dc in dcols:
if len(th.getColDesc(tempname, dc)) > 0:
datacolumn = dc
break
fpars['datacolumn'] = datacolumn
casalog.post('Will create a Multi-MS for: ' + myviso)
fpars['vis'] = tempname
fpars['flagbackup'] = False
fpars['outputvis'] = outputmms
fpars['separationaxis'] = separationaxis
fpars['numsubms'] = numsubms
# Run partition only at the MPIServers
pdh = ParallelDataHelper('partition', fpars)
# Get a cluster
pdh.setupCluster(thistask='partition')
try:
pdh.go()
# Remove original MS
shutil.rmtree(tempname)
except Exception, instance:
# Restore MS in case of error in MMS creation
shutil.move(tempname, myviso)
casalog.post('%s' % instance, 'ERROR')
return False
casalog.origin('importasdm')
# Create a .flagversions for the MS or MMS
if flagbackup:
for myviso in vistoproc:
if os.path.exists(myviso):
aflocal.open(myviso)
aflocal.saveflagversion(
'Original',
comment='Original flags at import into CASA',
merge='save')
aflocal.done()
# Importasdm Flag Parsing
if os.access(asdm + '/Flag.xml', os.F_OK):
# Find Flag.xml
casalog.post('Found Flag.xml in SDM')
# Find Antenna.xml
if os.access(asdm + '/Antenna.xml', os.F_OK):
casalog.post('Found Antenna.xml in SDM')
else:
raise Exception, 'Failed to find Antenna.xml in SDM'
# Find SpectralWindow.xml
if os.access(asdm + '/SpectralWindow.xml', os.F_OK):
casalog.post('Found SpectralWindow.xml in SDM')
else:
raise Exception, \
'Failed to find SpectralWindow.xml in SDM'
#
# Parse Flag.xml into flag dictionary
#
if process_flags:
flagcmds = fh.parseXML(asdm, float(tbuff))
onlinekeys = flagcmds.keys()
nflags = onlinekeys.__len__()
# Apply flags to the MS
if nflags > 0:
idx = 0
for myviso in vistoproc:
if applyflags:
# Open the MS and attach it to the tool
aflocal.open(myviso)
# Select the data
aflocal.selectdata()
# Setup the agent's parameters
fh.parseAgents(aflocal, flagcmds, [], True, True,
'')
# Initialize the agents
aflocal.init()
# Run the tool
aflocal.run(True, True)
casalog.post('Applied %s flag commands to %s' %
(nflags, myviso))
# Destroy the tool and de-attach the MS
aflocal.done()
# Save to FLAG_CMD table. APPLIED is set to True.
fh.writeFlagCommands(myviso, flagcmds, True, '',
'', True)
else:
casalog.post(
'Will not apply flags to %s (apply_flags=False), use flagcmd to apply'
% myviso)
# Write to FLAG_CMD, APPLIED is set to False
fh.writeFlagCommands(myviso, flagcmds, False, '',
'', True)
# Save the flag cmds to an ASCII file
if savecmds:
# Save to standard filename
fh.writeFlagCommands(myviso, flagcmds, False, '',
outfile[idx], False)
casalog.post('Saved %s flag commands to %s' %
(nflags, outfile[idx]))
idx += 1
else:
casalog.post('There are no flag commands to process')
else:
casalog.post('There is no Flag.xml in ASDM', 'WARN')
# Write parameters to HISTORY table of MS
mslocal = mstool()
param_names = importasdm.func_code.co_varnames[:importasdm.func_code.
co_argcount]
param_vals = [eval(p) for p in param_names]
for myviso in vistoproc:
write_history(mslocal, myviso, 'importasdm', param_names,
param_vals, casalog)
return
def importasdm(
asdm=None,
vis=None,
createmms=None,
separationaxis=None,
numsubms=None,
singledish=None,
antenna=None,
corr_mode=None,
srt=None,
time_sampling=None,
ocorr_mode=None,
compression=None,
lazy=None,
asis=None,
wvr_corrected_data=None,
scans=None,
ignore_time=None,
process_syspower=None,
process_caldevice=None,
process_pointing=None,
process_flags=None,
tbuff=None,
applyflags=None,
savecmds=None,
outfile=None,
flagbackup=None,
verbose=None,
overwrite=None,
showversion=None,
useversion=None,
bdfflags=None,
with_pointing_correction=None,
remove_ref_undef=None,
convert_ephem2geo=None
):
"""Convert an ALMA Science Data Model observation into a CASA visibility file (MS) or single-dish data format (Scantable).
The conversion of the ALMA SDM archive format into a measurement set. This version
is under development and is geared to handling many spectral windows of different
shapes.
Keyword arguments:
asdm -- Name of input ASDM file (directory)
default: none; example: asdm='ExecBlock3'
vis -- Root ms or scantable name, note a prefix (.ms or .asap) is NOT appended to this name
default: none
createmms -- Create a Multi-MS
default: False
singledish -- Set True to write data as single-dish format (Scantable)
default: False singledish expandable parameter
antenna -- antenna name or id.
corr_mode -- correlation mode to be considered on input. Could
be one or more of the following, ao, co, ac, or all
default: all
srt -- spectral resolution type. Could be one or more of
the following, fr, ca, bw, or all
default: all
time_sampling -- specifies the time sampling, INTEGRATION and/or
SUBINTEGRATION. could be one or more of the following
i, si, or all.
default: all
ocorr_mode -- output data for correlation mode AUTO_ONLY
(ao) or CROSS_ONLY (co) or CROSS_AND_AUTO (ca)
default: ca
compression -- produces comrpressed columns in the resulting measurement set.
default: False
lazy -- Make the MS DATA column read the ASDM Binary data directly
(faster import, smaller MS)
default: False
asis -- creates verbatim copies of the ASDM tables in
the output measurement set. The value given to
this option must be a list of table names separated
by space characters; the wildcard character '*' is
allowed in table names.
wvr_corrected_data -- specifies wich values are considered in the
ASDM binary data to fill the DATA column in
the MAIN table of the MS. Expected values for
this option are 'no' for the uncorrected data
(this is the default), 'yes' for the corrected
data and 'both' for corrected and uncorrected
data. In the latter case, two measurement sets
are created, one containing the uncorrected
data and the other one, whose name is suffixed
by '-wvr-corrected', containing the corrected
data.
scans -- processes only the scans specified in the option's value. This value is a semicolon
separated list of scan specifications. A scan specification consists in an exec bock index
followed by the character ':' followed by a comma separated list of scan indexes or scan
index ranges. A scan index is relative to the exec block it belongs to. Scan indexes are
1-based while exec blocks's are 0-based. "0:1" or "2:2~6" or "0:1,1:2~6,8;2:,3:24~30" "1,2"
are valid values for the option. "3:" alone will be interpreted as 'all the scans of the
exec block#3'. An scan index or a scan index range not preceded by an exec block index will
be interpreted as 'all the scans with such indexes in all the exec blocks'. By default
all the scans are considered.
ignore_time -- All the rows of the tables Feed, History, Pointing, Source, SysCal, CalDevice, SysPower,
and Weather are processed independently of the time range of the selected exec block / scan.
process_syspower -- The SysPower table is processed if and only if this parameter is set to True.
default: True
process_caldevice -- The CalDevice table is processed if and only if this parameter is set to True.
default: True
process_pointing -- The Pointing table is processed if and only if this parameter is set to True.
If the parameter is set to False the resulting MS will have an empty POINTING table.
default: True
process_flags -- Process the online flags and save them to the FLAG_CMD sub-table.
default: True
>>> process_flags expandable parameter
tbuff -- Time padding buffer (in seconds).
default: 0.0
applyflags -- Apply the online flags to the MS.
default: False
savecmds -- Save the online flags to an ASCII file.
default: False
outfile -- Filename to save the online flags.
default: ''
flagbackup -- Backup the FLAG column in the .flagversions.
default: True
verbose -- produce log output as asdm2MS is being run.
overwrite -- Over write an existing MS.
showversion -- report the version of the asdm2MS being used.
useversion -- Selects the version of asdm2MS to be used (presently only \'v3\' is available).
default: v3
bdfflags -- Set the MS FLAG column according to the ASDM _binary_ flags
default: false
with_pointing_correction -- add (ASDM::Pointing::encoder - ASDM::Pointing::pointingDirection)
to the value to be written in MS::Pointing::direction
default: false
remove_ref_undef -- if set to True then apply fixspwbackport on the resulting MSes.
convert_ephem2geo -- if True, convert any attached ephemerides to the GEO reference frame
"""
# Python script
# make agentflagger tool local
aflocal = casac.agentflagger()
# make table tool local
tblocal = casac.table()
try:
casalog.origin('importasdm')
viso = ''
visoc = '' # for the wvr corrected version, if needed
# -----------------------------------------
# beginning of importasdm_sd implementation
# -----------------------------------------
if singledish:
theexecutable = 'asdm2ASAP'
# if useversion == 'v2':
# theexecutable = 'oldasdm2ASAP'
if compression:
casalog.post('compression=True has no effect for single-dish format.')
cmd = 'which %s > /dev/null 2>&1' % theexecutable
ret = os.system(cmd)
if ret == 0:
import commands
casalog.post('found %s' % theexecutable)
if showversion:
execute_string = theexecutable + ' --help'
else:
execute_string = theexecutable + ' -asdm ' + asdm
if len(vis) != 0:
execute_string += ' -asap ' + vis.rstrip('/')
execute_string += ' -antenna ' + str(antenna) \
+ ' -apc ' + wvr_corrected_data \
+ ' -time-sampling ' + time_sampling.lower() \
+ ' -overwrite ' + str(overwrite)
if corr_mode == 'all':
execute_string += \
' -corr-mode ao,ca -ocorr-mode ao'
else:
execute_string += ' -corr-mode ' \
+ corr_mode.replace(' ', ',') \
+ ' -ocorr-mode ao'
if srt == 'all':
execute_string += ' -srt ' + srt
else:
execute_string += ' -srt ' + srt.replace(' ',
',')
execute_string += ' -logfile ' + casalog.logfile()
casalog.post('execute_string is')
casalog.post(' ' + execute_string)
ret = os.system(execute_string)
if ret != 0 and not showversion:
casalog.post(theexecutable
+ ' terminated with exit code '
+ str(ret), 'SEVERE')
# raise Exception, "ASDM conversion error, please check if it is a valid ASDM and/or useversion='%s' is consistent with input ASDM."%(useversion)
raise Exception, \
'ASDM conversion error, please check if it is a valid ASDM.'
else:
casalog.post('You have to build ASAP to be able to create single-dish data.'
, 'SEVERE')
# implementation of asis option using tb.fromASDM
if asis != '':
import commands
asdmTables = commands.getoutput('ls %s/*.xml'
% asdm).split('\n')
asdmTabNames = []
for tab in asdmTables:
asdmTabNames.append(tab.split('/')[-1].rstrip('.xml'
))
if asis == '*':
targetTables = asdmTables
targetTabNames = asdmTabNames
else:
targetTables = []
targetTabNames = []
tmpTabNames = asis.split()
for i in xrange(len(tmpTabNames)):
tab = tmpTabNames[i]
try:
targetTables.append(asdmTables[asdmTabNames.index(tab)])
targetTabNames.append(tab)
except:
pass
outTabNames = []
outTables = []
for tab in targetTabNames:
out = 'ASDM_' + tab.upper()
outTabNames.append(out)
outTables.append(vis + '/' + out)
tblocal.open(vis, nomodify=False)
wtb = casac.table()
for i in xrange(len(outTables)):
wtb.fromASDM(outTables[i], targetTables[i])
tblocal.putkeyword(outTabNames[i], 'Table: %s'
% outTables[i])
tblocal.flush()
tblocal.close()
return
# -----------------------------------
# end of importasdm_sd implementation
# -----------------------------------
if len(vis) > 0:
viso = vis
tmps = vis.rstrip('.ms')
if tmps == vis:
visoc = vis + '-wvr-corrected'
else:
visoc = tmps + '-wvr-corrected.ms'
if singledish:
viso = vis.rstrip('/') + '.importasdm.tmp.ms'
else:
viso = asdm.rstrip("/") + '.ms'
visoc = asdm.rstrip("/") + '-wvr-corrected.ms'
vis = asdm.rstrip("/")
if singledish:
viso = asdm.rstrip('/') + '.importasdm.tmp.ms'
vis = asdm.rstrip('/') + '.asap'
useversion = 'v3'
theexecutable = 'asdm2MS'
execute_string = theexecutable + ' --icm "' + corr_mode \
+ '" --isrt "' + srt + '" --its "' + time_sampling \
+ '" --ocm "' + ocorr_mode + '" --wvr-corrected-data "' \
+ wvr_corrected_data + '" --asis "' + asis \
+ '" --logfile "' + casalog.logfile() + '"'
if len(scans) > 0:
execute_string = execute_string + ' --scans ' + scans
if ignore_time:
execute_string = execute_string + ' --ignore-time'
if useversion == 'v3':
if not process_syspower:
execute_string = execute_string + ' --no-syspower'
if not process_caldevice:
execute_string = execute_string + ' --no-caldevice'
if not process_pointing:
execute_string = execute_string + ' --no-pointing'
if compression:
execute_string = execute_string + ' --compression'
elif lazy:
execute_string = execute_string + ' --lazy'
if verbose:
execute_string = execute_string + ' --verbose'
# if not overwrite and os.path.exists(viso):
# raise Exception, \
# 'You have specified an existing MS and have indicated you do not wish to overwrite it'
# Compression
if compression:
# viso = viso + '.compressed'
viso = viso.rstrip('.ms') + '.compressed.ms'
visoc = visoc.rstrip('.ms') + '.compressed.ms'
vistoproc = [] # the output MSs to post-process
if wvr_corrected_data == 'no' or wvr_corrected_data == 'both':
vistoproc.append(viso)
if (wvr_corrected_data == 'yes' or wvr_corrected_data == 'both') :
vistoproc.append(visoc)
for ff in vistoproc:
if not overwrite and os.path.exists(ff):
raise Exception, \
'You have specified an existing MS and have indicated you do not wish to overwrite it: %s'%ff
# If viso+".flagversions" then process differently depending on the value of overwrite..
#
if flagbackup:
for myviso in vistoproc:
dotFlagversion = myviso + '.flagversions'
if os.path.exists(dotFlagversion):
if overwrite:
casalog.post("Found '" + dotFlagversion
+ "' . It'll be deleted before running the filler."
)
os.system('rm -rf %s' % dotFlagversion)
else:
casalog.post("Found '%s' but can't overwrite it."
% dotFlagversion)
raise Exception, "Found '%s' but can't overwrite it." \
% dotFlagversion
# Make outfile always a list
if isinstance(outfile, str):
if outfile == '':
outfile = []
else:
noutfile = [outfile]
outfile = noutfile
if savecmds:
if len(outfile) == 0:
# Create default names for the online flags
for myviso in vistoproc:
outfile.append(myviso.replace('.ms','_cmd.txt'))
elif len(outfile) != len(vistoproc):
casalog.post('List of outfile names does not match list of MSs','WARN')
casalog.post('Will save online flags to temporary filenames', 'WARN')
outfile = []
for myviso in vistoproc:
online_file = myviso.replace('.ms','_TEMP_cmd.txt')
outfile.append(online_file)
if not overwrite:
for of in outfile:
if os.path.exists(of):
raise Exception, "Cannot overwrite online flags file '%s'; overwrite is set to False."% of
execute_string = execute_string + ' ' + asdm + ' ' + viso
if showversion:
casalog.post("You set option \'showversion\' to True. Will just output the version information and then terminate."
, 'WARN')
execute_string = theexecutable + ' --revision'
if with_pointing_correction:
execute_string = execute_string + ' --with-pointing-correction'
casalog.post('Running ' + theexecutable
+ ' standalone invoked as:')
# print execute_string
casalog.post(execute_string)
exitcode = os.system(execute_string)
if exitcode != 0:
if not showversion:
casalog.post(theexecutable
+ ' terminated with exit code '
+ str(exitcode), 'SEVERE')
raise Exception, \
'ASDM conversion error. Please check if it is a valid ASDM and that data/alma/asdm is up to date.'
if showversion:
return
#
# Possibly remove the element the name of the measurement set expected to contain the corrected data from the list of of produced measurement
# sets if it appears the filler did not find any corrected data.
#
if not os.path.exists(visoc):
vistoproc = [myviso for myviso in vistoproc if myviso != visoc]
# CAS-7369. HISTORY should be written after createmms is tested
#
# Populate the HISTORY table of the MS with information about the context in which it's been created
#
try:
mslocal = mstool()
param_names = importasdm.func_code.co_varnames[:importasdm.func_code.co_argcount]
param_vals = [eval(p) for p in param_names]
for myviso in vistoproc:
write_history(mslocal, myviso, 'importasdm', param_names,
param_vals, casalog)
except Exception, instance:
casalog.post("*** Error \'%s\' updating HISTORY" % (instance),
'WARN')
return False
if mslocal:
mslocal = None
#
# Do we apply fixspwbackport
if remove_ref_undef :
casalog.post('remove_ref_undef=True: fixspwbackport will be applied ...')
for myviso in vistoproc:
cmd = 'fixspwbackport ' + myviso
casalog.post('Running fixspwbackport standalone invoked as:')
casalog.post(cmd)
cmdexitcode = os.system(cmd)
if cmdexitcode != 0:
casalog.post(cmd
+ ' terminated with exit code '
+ str(cmdexitcode), 'SEVERE')
raise Exception, 'fixspwbackport error.'
# Binary Flag processing
if bdfflags:
casalog.post('Parameter bdfflags==True: flags from the ASDM binary data will be used to set the MS flags ...')
bdffexecutable = 'bdflags2MS '
bdffexecstring_base = bdffexecutable+' -f ALL'
if len(scans) > 0:
bdffexecstring_base = bdffexecstring_base + ' --scans ' + scans
for myviso in vistoproc:
if myviso.find("wvr-corrected") != -1:
options = " --wvr-corrected=True "
else:
options = " "
bdffexecstring = bdffexecstring_base + options + asdm + ' ' + myviso
casalog.post('Running '+bdffexecutable+' standalone invoked as:')
casalog.post(bdffexecstring)
bdffexitcode = os.system(bdffexecstring)
if bdffexitcode != 0:
casalog.post(bdffexecutable
+ ' terminated with exit code '
+ str(bdffexitcode), 'SEVERE')
raise Exception, \
'ASDM binary flags conversion error. Please check if it is a valid ASDM and that data/alma/asdm is up to date.'
for myviso in vistoproc:
if os.path.exists(myviso) and flagbackup==True:
aflocal.open(myviso)
aflocal.saveflagversion('Original',
comment='Original flags at import into CASA',
merge='save')
aflocal.done()
# Importasdm Flag Parsing
if os.access(asdm + '/Flag.xml', os.F_OK):
# Find Flag.xml
casalog.post('Found Flag.xml in SDM')
# Find Antenna.xml
if os.access(asdm + '/Antenna.xml', os.F_OK):
casalog.post('Found Antenna.xml in SDM')
else:
raise Exception, 'Failed to find Antenna.xml in SDM'
# Find SpectralWindow.xml
if os.access(asdm + '/SpectralWindow.xml', os.F_OK):
casalog.post('Found SpectralWindow.xml in SDM')
else:
raise Exception, \
'Failed to find SpectralWindow.xml in SDM'
#
# Parse Flag.xml into flag dictionary
#
if process_flags:
flagcmds = fh.parseXML(asdm, float(tbuff))
onlinekeys = flagcmds.keys()
nflags = onlinekeys.__len__()
# Apply flags to the MS
if nflags > 0:
idx = 0
for myviso in vistoproc:
if applyflags:
# Open the MS and attach it to the tool
aflocal.open(myviso)
# Select the data
aflocal.selectdata()
# Setup the agent's parameters
fh.parseAgents(aflocal, flagcmds, [], True, True, '')
# Initialize the agents
aflocal.init()
# Run the tool
aflocal.run(True, True)
casalog.post('Applied %s flag commands to %s'%(nflags,myviso))
# Destroy the tool and de-attach the MS
aflocal.done()
# Save to FLAG_CMD table. APPLIED is set to True.
fh.writeFlagCommands(myviso, flagcmds, True, '', '', True)
else:
casalog.post('Will not apply flags to %s (apply_flags=False), use flagcmd to apply'%myviso)
# Write to FLAG_CMD, APPLIED is set to False
fh.writeFlagCommands(myviso, flagcmds, False, '', '', True)
# Save the flag cmds to an ASCII file
if savecmds:
# Save to standard filename
fh.writeFlagCommands(myviso, flagcmds, False, '', outfile[idx], False)
casalog.post('Saved %s flag commands to %s'%(nflags,outfile[idx]))
idx += 1
else:
casalog.post('There are no flag commands to process')
else:
casalog.post('There is no Flag.xml in ASDM', 'WARN')
import recipes.ephemerides.convertephem as ce
theephemfields = ce.findattachedephemfields(myviso,field='*')
if len(theephemfields)>0: # temporary fix until asdm2MS does this internally: recalc the UVW coordinates for ephem fields
imt = imtool()
imt.open(myviso, usescratch=False)
imt.calcuvw(theephemfields, refcode='J2000', reuse=False)
imt.close()
if convert_ephem2geo:
for myviso in vistoproc:
ce.convert2geo(myviso, '*') # convert any attached ephemerides to GEO
# CAS-7369 - Create an output Multi-MS (MMS)
if createmms:
# Get the default parameters of partition
from tasks import partition
fpars = partition.parameters
for mypar in fpars.keys():
fpars[mypar] = partition.itsdefault(mypar)
# Call the cluster for each MS
for myviso in vistoproc:
casalog.origin('importasdm')
outputmms = myviso.replace('.ms','.mms')
# Get the proper column
datacolumn = 'DATA'
dcols = ['DATA', 'FLOAT_DATA']
for dc in dcols:
if len(th.getColDesc(myviso, dc)) > 0:
datacolumn = dc
break
fpars['datacolumn'] = datacolumn
casalog.post('Will create a Multi-MS for: '+myviso)
fpars['vis'] = myviso
fpars['flagbackup'] = False
fpars['outputvis'] = outputmms
fpars['separationaxis'] = separationaxis
fpars['numsubms'] = numsubms
pdh = ParallelDataHelper('partition', fpars)
# Get a cluster
pdh.setupCluster(thistask='partition')
try:
pdh.go()
# Rename MMS to MS
shutil.rmtree(myviso)
shutil.move(outputmms, myviso)
except Exception, instance:
casalog.post('%s'%instance,'ERROR')
return False
casalog.origin('importasdm')
return