#Copyright (C) 2019 Inter-University Institute for Data Intensive Astronomy
#See processMeerKAT.py for license details.
import os
import config_parser
from config_parser import validate_args as va
from cal_scripts import bookkeeping
import glob
PLOT_DIR = 'plots'
EXTN = 'pdf'
# CASA imports
from taskinit import *
from tasks import *
from casac import casac
msmd = casac.msmetadata()
def sort_by_antenna(fname):
"""Sort list of plot files by antenna number with format "PLOT_DIR/caltype_xaxis_yaxis_antXX~YY.EXTN",
to be passed into key parameter of list.sort.
Arguments:
----------
fname : str
Filename of plot file.
Returns:
--------
ant : int
Antenna numbers."""
def importasdm(asdm=None,
vis=None,
createmms=None,
separationaxis=None,
numsubms=None,
corr_mode=None,
srt=None,
time_sampling=None,
ocorr_mode=None,
compression=None,
lazy=None,
asis=None,
wvr_corrected_data=None,
scans=None,
ignore_time=None,
process_syspower=None,
process_caldevice=None,
process_pointing=None,
process_flags=None,
tbuff=None,
applyflags=None,
savecmds=None,
outfile=None,
flagbackup=None,
verbose=None,
overwrite=None,
showversion=None,
useversion=None,
bdfflags=None,
with_pointing_correction=None,
remove_ref_undef=None,
convert_ephem2geo=None,
polyephem_tabtimestep=None):
"""Convert an ALMA Science Data Model observation into a CASA visibility file (MS) or single-dish data format (Scantable).
The conversion of the ALMA SDM archive format into a measurement set. This version
is under development and is geared to handling many spectral windows of different
shapes.
Keyword arguments:
asdm -- Name of input ASDM file (directory)
default: none; example: asdm='ExecBlock3'
vis -- Root ms or scantable name, note a prefix (.ms or .asap) is NOT appended to this name
default: none
createmms -- Create a Multi-MS
default: False
corr_mode -- correlation mode to be considered on input. Could
be one or more of the following, ao, co, ac, or all
default: all
srt -- spectral resolution type. Could be one or more of
the following, fr, ca, bw, or all
default: all
time_sampling -- specifies the time sampling, INTEGRATION and/or
SUBINTEGRATION. could be one or more of the following
i, si, or all.
default: all
ocorr_mode -- output data for correlation mode AUTO_ONLY
(ao) or CROSS_ONLY (co) or CROSS_AND_AUTO (ca)
default: ca
compression -- produces comrpressed columns in the resulting measurement set.
default: False
lazy -- Make the MS DATA column read the ASDM Binary data directly
(faster import, smaller MS)
default: False
asis -- creates verbatim copies of the ASDM tables in
the output measurement set. The value given to
this option must be a list of table names separated
by space characters; the wildcard character '*' is
allowed in table names.
wvr_corrected_data -- specifies wich values are considered in the
ASDM binary data to fill the DATA column in
the MAIN table of the MS. Expected values for
this option are 'no' for the uncorrected data
(this is the default), 'yes' for the corrected
data and 'both' for corrected and uncorrected
data. In the latter case, two measurement sets
are created, one containing the uncorrected
data and the other one, whose name is suffixed
by '-wvr-corrected', containing the corrected
data.
scans -- processes only the scans specified in the option's value. This value is a semicolon
separated list of scan specifications. A scan specification consists in an exec bock index
followed by the character ':' followed by a comma separated list of scan indexes or scan
index ranges. A scan index is relative to the exec block it belongs to. Scan indexes are
1-based while exec blocks's are 0-based. "0:1" or "2:2~6" or "0:1,1:2~6,8;2:,3:24~30" "1,2"
are valid values for the option. "3:" alone will be interpreted as 'all the scans of the
exec block#3'. An scan index or a scan index range not preceded by an exec block index will
be interpreted as 'all the scans with such indexes in all the exec blocks'. By default
all the scans are considered.
ignore_time -- All the rows of the tables Feed, History, Pointing, Source, SysCal, CalDevice, SysPower,
and Weather are processed independently of the time range of the selected exec block / scan.
process_syspower -- The SysPower table is processed if and only if this parameter is set to True.
default: True
process_caldevice -- The CalDevice table is processed if and only if this parameter is set to True.
default: True
process_pointing -- The Pointing table is processed if and only if this parameter is set to True.
If the parameter is set to False the resulting MS will have an empty POINTING table.
default: True
process_flags -- Process the online flags and save them to the FLAG_CMD sub-table.
default: True
>>> process_flags expandable parameter
tbuff -- Time padding buffer (in seconds).
default: 0.0
applyflags -- Apply the online flags to the MS.
default: False
savecmds -- Save the online flags to an ASCII file.
default: False
outfile -- Filename to save the online flags.
default: ''
flagbackup -- Backup the FLAG column in the .flagversions.
default: True
verbose -- produce log output as asdm2MS is being run.
overwrite -- Over write an existing MS.
showversion -- report the version of the asdm2MS being used.
useversion -- Selects the version of asdm2MS to be used (presently only \'v3\' is available).
default: v3
bdfflags -- Set the MS FLAG column according to the ASDM _binary_ flags
default: false
with_pointing_correction -- add (ASDM::Pointing::encoder - ASDM::Pointing::pointingDirection)
to the value to be written in MS::Pointing::direction
default: false
remove_ref_undef -- if set to True then apply fixspwbackport on the resulting MSes.
convert_ephem2geo -- if True, convert any attached ephemerides to the GEO reference frame
polyephem_tabtimestep -- Timestep (days) for the tabulation of polynomial ephemerides. A value <= 0 disables tabulation.
Presently, VLA data can contain polynomial ephemerides. ALMA data uses tabulated values.
default: 0.
"""
# Python script
# make agentflagger tool local
aflocal = casac.agentflagger()
# make table tool local
tblocal = casac.table()
try:
casalog.origin('importasdm')
viso = ''
visoc = '' # for the wvr corrected version, if needed
if len(vis) > 0:
viso = vis
tmps = vis.rstrip('.ms')
if tmps == vis:
visoc = vis + '-wvr-corrected'
else:
visoc = tmps + '-wvr-corrected.ms'
else:
viso = asdm.rstrip("/") + '.ms'
visoc = asdm.rstrip("/") + '-wvr-corrected.ms'
vis = asdm.rstrip("/")
useversion = 'v3'
theexecutable = 'asdm2MS'
execute_string = theexecutable + ' --icm "' + corr_mode \
+ '" --isrt "' + srt + '" --its "' + time_sampling \
+ '" --ocm "' + ocorr_mode + '" --wvr-corrected-data "' \
+ wvr_corrected_data + '" --asis "' + asis \
+ '" --logfile "' + casalog.logfile() + '"'
if len(scans) > 0:
execute_string = execute_string + ' --scans ' + scans
if ignore_time:
execute_string = execute_string + ' --ignore-time'
if useversion == 'v3':
if not process_syspower:
execute_string = execute_string + ' --no-syspower'
if not process_caldevice:
execute_string = execute_string + ' --no-caldevice'
if not process_pointing:
execute_string = execute_string + ' --no-pointing'
if compression:
execute_string = execute_string + ' --compression'
elif lazy:
execute_string = execute_string + ' --lazy'
if verbose:
execute_string = execute_string + ' --verbose'
# if not overwrite and os.path.exists(viso):
# raise Exception, \
# 'You have specified an existing MS and have indicated you do not wish to overwrite it'
# Compression
if compression:
# viso = viso + '.compressed'
viso = viso.rstrip('.ms') + '.compressed.ms'
visoc = visoc.rstrip('.ms') + '.compressed.ms'
vistoproc = [] # the output MSs to post-process
if wvr_corrected_data == 'no' or wvr_corrected_data == 'both':
vistoproc.append(viso)
if (wvr_corrected_data == 'yes' or wvr_corrected_data == 'both'):
vistoproc.append(visoc)
for ff in vistoproc:
if not overwrite and os.path.exists(ff):
raise Exception, \
'You have specified an existing MS and have indicated you do not wish to overwrite it: %s'%ff
# If viso+".flagversions" then process differently depending on the value of overwrite..
#
if flagbackup:
for myviso in vistoproc:
dotFlagversion = myviso + '.flagversions'
if os.path.exists(dotFlagversion):
if overwrite:
casalog.post(
"Found '" + dotFlagversion +
"' . It'll be deleted before running the filler.")
os.system('rm -rf %s' % dotFlagversion)
else:
casalog.post("Found '%s' but can't overwrite it." %
dotFlagversion)
raise Exception, "Found '%s' but can't overwrite it." \
% dotFlagversion
# Make outfile always a list
if isinstance(outfile, str):
if outfile == '':
outfile = []
else:
noutfile = [outfile]
outfile = noutfile
if savecmds:
if len(outfile) == 0:
# Create default names for the online flags
for myviso in vistoproc:
outfile.append(myviso.replace('.ms', '_cmd.txt'))
elif len(outfile) != len(vistoproc):
casalog.post(
'List of outfile names does not match list of MSs', 'WARN')
casalog.post('Will save online flags to temporary filenames',
'WARN')
outfile = []
for myviso in vistoproc:
online_file = myviso.replace('.ms', '_TEMP_cmd.txt')
outfile.append(online_file)
if not overwrite:
for of in outfile:
if os.path.exists(of):
raise Exception, "Cannot overwrite online flags file '%s'; overwrite is set to False." % of
execute_string = execute_string + ' ' + asdm + ' ' + viso
if showversion:
casalog.post(
"You set option \'showversion\' to True. Will just output the version information and then terminate.",
'WARN')
execute_string = theexecutable + ' --revision'
if with_pointing_correction:
execute_string = execute_string + ' --with-pointing-correction'
if (polyephem_tabtimestep !=
None) and (type(polyephem_tabtimestep) == int
or type(polyephem_tabtimestep) == float):
if polyephem_tabtimestep > 0:
casalog.post(
'Will tabulate all attached polynomial ephemerides with a time step of '
+ str(polyephem_tabtimestep) + ' days.')
if polyephem_tabtimestep > 1.:
casalog.post(
'A tabulation timestep of <= 1 days is recommended.',
'WARN')
execute_string = execute_string + ' --polyephem-tabtimestep ' + str(
polyephem_tabtimestep)
casalog.post('Running ' + theexecutable + ' standalone invoked as:')
# print execute_string
casalog.post(execute_string)
exitcode = os.system(execute_string)
if exitcode != 0:
if not showversion:
casalog.post(
theexecutable + ' terminated with exit code ' +
str(exitcode), 'SEVERE')
raise Exception, \
'ASDM conversion error. Please check if it is a valid ASDM and that data/alma/asdm is up to date.'
if showversion:
return
#
# Possibly remove the name of the measurement set expected to contain the corrected data from the list of of produced measurement
# sets if it appears the filler did not find any corrected data.
#
if not os.path.exists(visoc):
vistoproc = [myviso for myviso in vistoproc if myviso != visoc]
# CAS-7369. HISTORY should be written after createmms is tested
#
# Populate the HISTORY table of the MS with information about the context in which it's been created
#
try:
mslocal = mstool()
param_names = importasdm.func_code.co_varnames[:importasdm.
func_code.
co_argcount]
param_vals = [eval(p) for p in param_names]
for myviso in vistoproc:
write_history(mslocal, myviso, 'importasdm', param_names,
param_vals, casalog)
except Exception, instance:
casalog.post("*** Error \'%s\' updating HISTORY" % (instance),
'WARN')
return False
if mslocal:
mslocal = None
#
# Do we apply fixspwbackport
if remove_ref_undef:
casalog.post(
'remove_ref_undef=True: fixspwbackport will be applied ...')
for myviso in vistoproc:
cmd = 'fixspwbackport ' + myviso
casalog.post('Running fixspwbackport standalone invoked as:')
casalog.post(cmd)
cmdexitcode = os.system(cmd)
if cmdexitcode != 0:
casalog.post(
cmd + ' terminated with exit code ' + str(cmdexitcode),
'SEVERE')
raise Exception, 'fixspwbackport error.'
# Binary Flag processing
if bdfflags:
casalog.post(
'Parameter bdfflags==True: flags from the ASDM binary data will be used to set the MS flags ...'
)
bdffexecutable = 'bdflags2MS '
bdffexecstring_base = bdffexecutable + ' -f ALL' + ' --ocm "' + ocorr_mode \
+ '" --logfile "' + casalog.logfile() + '"'
if len(scans) > 0:
bdffexecstring_base = bdffexecstring_base + ' --scans ' + scans
if lazy and not compression:
bdffexecstring_base = bdffexecstring_base + ' --lazy=true'
for myviso in vistoproc:
if myviso.find("wvr-corrected") != -1:
options = " --wvr-corrected=True "
else:
options = " "
bdffexecstring = bdffexecstring_base + options + asdm + ' ' + myviso
casalog.post('Running ' + bdffexecutable +
' standalone invoked as:')
casalog.post(bdffexecstring)
bdffexitcode = os.system(bdffexecstring)
if bdffexitcode != 0:
casalog.post(
bdffexecutable + ' terminated with exit code ' +
str(bdffexitcode), 'SEVERE')
raise Exception, \
'ASDM binary flags conversion error. Please check if it is a valid ASDM and that data/alma/asdm is up to date.'
theephemfields = ce.findattachedephemfields(myviso, field='*')
if len(theephemfields) > 0:
# until asdm2MS does this internally: recalc the UVW coordinates for ephem fields
imt = imtool()
imt.open(myviso, usescratch=False)
imt.calcuvw(theephemfields, refcode='J2000', reuse=False)
imt.close()
if convert_ephem2geo:
for myviso in vistoproc:
ce.convert2geo(myviso,
'*') # convert any attached ephemerides to GEO
if len(theephemfields) > 0:
# also set the direction column in the SOURCE table
tblocal.open(myviso + '/FIELD', nomodify=False)
sourceids = tblocal.getcol('SOURCE_ID')
ftimes = tblocal.getcol('TIME')
ftimekw = tblocal.getcolkeywords('TIME')
tmpa = tblocal.getcol('PHASE_DIR')
origphasedir = tmpa
affectedsids = []
thesamplefields = []
for fld in theephemfields: # determine all source ids used by the ephem fields
if not (sourceids[fld]
in affectedsids): # this source id wasn't handled yet
affectedsids.append(sourceids[fld])
thesamplefields.append(fld)
# need to temporarily change the offset (not all mosaics have an element at (0,0))
tmpa[0][0][fld] = 0.
tmpa[1][0][fld] = 0.
#endif
#endfor
tblocal.putcol('PHASE_DIR', tmpa)
tblocal.close()
tblocal.open(myviso + '/SOURCE')
sourceposref = tblocal.getcolkeywords(
'DIRECTION')['MEASINFO']['Ref']
tblocal.close()
directions = []
msmdlocal = casac.msmetadata()
msmdlocal.open(myviso)
for fld in thesamplefields:
thedirmeas = msmdlocal.phasecenter(fld)
if thedirmeas['refer'] != sourceposref:
casalog.post(
'Ephemeris is in ' + thedirmeas['refer'] +
' instead of ' + sourceposref +
' frame.\nEntry in SOURCE table will be converted to '
+ sourceposref, 'WARN')
melocal = metool()
melocal.doframe(thedirmeas)
thedirmeas = melocal.measure(thedirmeas, sourceposref)
directions.append(
[thedirmeas['m0']['value'], thedirmeas['m1']['value']])
thetime = me.epoch(v0=str(ftimes[fld]) + 's',
rf=ftimekw['MEASINFO']['Ref'])
casalog.post("Will set SOURCE direction for SOURCE_ID " +
str(sourceids[fld]) +
" to ephemeris phase center for time " +
str(thetime['m0']['value']) + " " +
thetime['m0']['unit'] + " " + thetime['refer'])
#endfor
msmdlocal.close()
# restore original PHASE_DIR
tblocal.open(myviso + '/FIELD', nomodify=False)
tblocal.putcol('PHASE_DIR', origphasedir)
tblocal.close()
# write source directions
tblocal.open(myviso + '/SOURCE', nomodify=False)
ssourceids = tblocal.getcol('SOURCE_ID')
sdirs = tblocal.getcol('DIRECTION')
for row in xrange(0, len(ssourceids)):
for i in xrange(0, len(affectedsids)):
if ssourceids[row] == affectedsids[i]:
sdirs[0][row] = directions[i][0]
sdirs[1][row] = directions[i][1]
break
#endfor
#endfor
tblocal.putcol('DIRECTION',
sdirs) # write back corrected directions
tblocal.close()
#end if
##############################################################################################3
# CAS-7369 - Create an output Multi-MS (MMS)
if createmms:
# Get the default parameters of partition
from tasks import partition
fpars = partition.parameters
for mypar in fpars.keys():
fpars[mypar] = partition.itsdefault(mypar)
# Call the cluster for each MS
for myviso in vistoproc:
casalog.origin('importasdm')
# Move original MS to tempdir
tempname = myviso + '.temp.ms'
outputmms = myviso
shutil.move(myviso, tempname)
# Get the proper column
datacolumn = 'DATA'
dcols = ['DATA', 'FLOAT_DATA']
for dc in dcols:
if len(th.getColDesc(tempname, dc)) > 0:
datacolumn = dc
break
fpars['datacolumn'] = datacolumn
casalog.post('Will create a Multi-MS for: ' + myviso)
fpars['vis'] = tempname
fpars['flagbackup'] = False
fpars['outputvis'] = outputmms
fpars['separationaxis'] = separationaxis
fpars['numsubms'] = numsubms
pdh = ParallelDataHelper('partition', fpars)
# Get a cluster
pdh.setupCluster(thistask='partition')
try:
pdh.go()
# Remove original MS
shutil.rmtree(tempname)
except Exception, instance:
# Restore MS in case of error in MMS creation
shutil.move(tempname, myviso)
casalog.post('%s' % instance, 'ERROR')
return False
casalog.origin('importasdm')
def importasdm(
asdm=None,
vis=None,
createmms=None,
separationaxis=None,
numsubms=None,
corr_mode=None,
srt=None,
time_sampling=None,
ocorr_mode=None,
compression=None,
lazy=None,
asis=None,
wvr_corrected_data=None,
scans=None,
ignore_time=None,
process_syspower=None,
process_caldevice=None,
process_pointing=None,
process_flags=None,
tbuff=None,
applyflags=None,
savecmds=None,
outfile=None,
flagbackup=None,
verbose=None,
overwrite=None,
showversion=None,
useversion=None,
bdfflags=None,
with_pointing_correction=None,
remove_ref_undef=None,
convert_ephem2geo=None,
polyephem_tabtimestep=None
):
"""Convert an ALMA Science Data Model observation into a CASA visibility file (MS) or single-dish data format (Scantable).
The conversion of the ALMA SDM archive format into a measurement set. This version
is under development and is geared to handling many spectral windows of different
shapes.
Keyword arguments:
asdm -- Name of input ASDM file (directory)
default: none; example: asdm='ExecBlock3'
vis -- Root ms or scantable name, note a prefix (.ms or .asap) is NOT appended to this name
default: none
createmms -- Create a Multi-MS
default: False
corr_mode -- correlation mode to be considered on input. Could
be one or more of the following, ao, co, ac, or all
default: all
srt -- spectral resolution type. Could be one or more of
the following, fr, ca, bw, or all
default: all
time_sampling -- specifies the time sampling, INTEGRATION and/or
SUBINTEGRATION. could be one or more of the following
i, si, or all.
default: all
ocorr_mode -- output data for correlation mode AUTO_ONLY
(ao) or CROSS_ONLY (co) or CROSS_AND_AUTO (ca)
default: ca
compression -- produces comrpressed columns in the resulting measurement set.
default: False
lazy -- Make the MS DATA column read the ASDM Binary data directly
(faster import, smaller MS)
default: False
asis -- creates verbatim copies of the ASDM tables in
the output measurement set. The value given to
this option must be a list of table names separated
by space characters; the wildcard character '*' is
allowed in table names.
wvr_corrected_data -- specifies wich values are considered in the
ASDM binary data to fill the DATA column in
the MAIN table of the MS. Expected values for
this option are 'no' for the uncorrected data
(this is the default), 'yes' for the corrected
data and 'both' for corrected and uncorrected
data. In the latter case, two measurement sets
are created, one containing the uncorrected
data and the other one, whose name is suffixed
by '-wvr-corrected', containing the corrected
data.
scans -- processes only the scans specified in the option's value. This value is a semicolon
separated list of scan specifications. A scan specification consists in an exec bock index
followed by the character ':' followed by a comma separated list of scan indexes or scan
index ranges. A scan index is relative to the exec block it belongs to. Scan indexes are
1-based while exec blocks's are 0-based. "0:1" or "2:2~6" or "0:1,1:2~6,8;2:,3:24~30" "1,2"
are valid values for the option. "3:" alone will be interpreted as 'all the scans of the
exec block#3'. An scan index or a scan index range not preceded by an exec block index will
be interpreted as 'all the scans with such indexes in all the exec blocks'. By default
all the scans are considered.
ignore_time -- All the rows of the tables Feed, History, Pointing, Source, SysCal, CalDevice, SysPower,
and Weather are processed independently of the time range of the selected exec block / scan.
process_syspower -- The SysPower table is processed if and only if this parameter is set to True.
default: True
process_caldevice -- The CalDevice table is processed if and only if this parameter is set to True.
default: True
process_pointing -- The Pointing table is processed if and only if this parameter is set to True.
If the parameter is set to False the resulting MS will have an empty POINTING table.
default: True
process_flags -- Process the online flags and save them to the FLAG_CMD sub-table.
default: True
>>> process_flags expandable parameter
tbuff -- Time padding buffer (in seconds).
default: 0.0
applyflags -- Apply the online flags to the MS.
default: False
savecmds -- Save the online flags to an ASCII file.
default: False
outfile -- Filename to save the online flags.
default: ''
flagbackup -- Backup the FLAG column in the .flagversions.
default: True
verbose -- produce log output as asdm2MS is being run.
overwrite -- Over write an existing MS.
showversion -- report the version of the asdm2MS being used.
useversion -- Selects the version of asdm2MS to be used (presently only \'v3\' is available).
default: v3
bdfflags -- Set the MS FLAG column according to the ASDM _binary_ flags
default: false
with_pointing_correction -- add (ASDM::Pointing::encoder - ASDM::Pointing::pointingDirection)
to the value to be written in MS::Pointing::direction
default: false
remove_ref_undef -- if set to True then apply fixspwbackport on the resulting MSes.
convert_ephem2geo -- if True, convert any attached ephemerides to the GEO reference frame
polyephem_tabtimestep -- Timestep (days) for the tabulation of polynomial ephemerides. A value <= 0 disables tabulation.
Presently, VLA data can contain polynomial ephemerides. ALMA data uses tabulated values.
default: 0.
"""
# Python script
# make agentflagger tool local
aflocal = casac.agentflagger()
# make table tool local
tblocal = casac.table()
try:
casalog.origin('importasdm')
viso = ''
visoc = '' # for the wvr corrected version, if needed
if len(vis) > 0:
viso = vis
tmps = vis.rstrip('.ms')
if tmps == vis:
visoc = vis + '-wvr-corrected'
else:
visoc = tmps + '-wvr-corrected.ms'
else:
viso = asdm.rstrip("/") + '.ms'
visoc = asdm.rstrip("/") + '-wvr-corrected.ms'
vis = asdm.rstrip("/")
useversion = 'v3'
theexecutable = 'asdm2MS'
execute_string = theexecutable + ' --icm "' + corr_mode \
+ '" --isrt "' + srt + '" --its "' + time_sampling \
+ '" --ocm "' + ocorr_mode + '" --wvr-corrected-data "' \
+ wvr_corrected_data + '" --asis "' + asis \
+ '" --logfile "' + casalog.logfile() + '"'
if len(scans) > 0:
execute_string = execute_string + ' --scans ' + scans
if ignore_time:
execute_string = execute_string + ' --ignore-time'
if useversion == 'v3':
if not process_syspower:
execute_string = execute_string + ' --no-syspower'
if not process_caldevice:
execute_string = execute_string + ' --no-caldevice'
if not process_pointing:
execute_string = execute_string + ' --no-pointing'
if compression:
execute_string = execute_string + ' --compression'
elif lazy:
execute_string = execute_string + ' --lazy'
if verbose:
execute_string = execute_string + ' --verbose'
# if not overwrite and os.path.exists(viso):
# raise Exception, \
# 'You have specified an existing MS and have indicated you do not wish to overwrite it'
# Compression
if compression:
# viso = viso + '.compressed'
viso = viso.rstrip('.ms') + '.compressed.ms'
visoc = visoc.rstrip('.ms') + '.compressed.ms'
vistoproc = [] # the output MSs to post-process
if wvr_corrected_data == 'no' or wvr_corrected_data == 'both':
vistoproc.append(viso)
if (wvr_corrected_data == 'yes' or wvr_corrected_data == 'both') :
vistoproc.append(visoc)
for ff in vistoproc:
if not overwrite and os.path.exists(ff):
raise Exception, \
'You have specified an existing MS and have indicated you do not wish to overwrite it: %s'%ff
# If viso+".flagversions" then process differently depending on the value of overwrite..
#
if flagbackup:
for myviso in vistoproc:
dotFlagversion = myviso + '.flagversions'
if os.path.exists(dotFlagversion):
if overwrite:
casalog.post("Found '" + dotFlagversion
+ "' . It'll be deleted before running the filler."
)
os.system('rm -rf %s' % dotFlagversion)
else:
casalog.post("Found '%s' but can't overwrite it."
% dotFlagversion)
raise Exception, "Found '%s' but can't overwrite it." \
% dotFlagversion
# Make outfile always a list
if isinstance(outfile, str):
if outfile == '':
outfile = []
else:
noutfile = [outfile]
outfile = noutfile
if savecmds:
if len(outfile) == 0:
# Create default names for the online flags
for myviso in vistoproc:
outfile.append(myviso.replace('.ms','_cmd.txt'))
elif len(outfile) != len(vistoproc):
casalog.post('List of outfile names does not match list of MSs','WARN')
casalog.post('Will save online flags to temporary filenames', 'WARN')
outfile = []
for myviso in vistoproc:
online_file = myviso.replace('.ms','_TEMP_cmd.txt')
outfile.append(online_file)
if not overwrite:
for of in outfile:
if os.path.exists(of):
raise Exception, "Cannot overwrite online flags file '%s'; overwrite is set to False."% of
execute_string = execute_string + ' ' + asdm + ' ' + viso
if showversion:
casalog.post("You set option \'showversion\' to True. Will just output the version information and then terminate."
, 'WARN')
execute_string = theexecutable + ' --revision'
if with_pointing_correction:
execute_string = execute_string + ' --with-pointing-correction'
if (polyephem_tabtimestep!=None) and (type(polyephem_tabtimestep)==int or type(polyephem_tabtimestep)==float):
if polyephem_tabtimestep>0:
casalog.post('Will tabulate all attached polynomial ephemerides with a time step of '
+str(polyephem_tabtimestep)+' days.')
if polyephem_tabtimestep>1.:
casalog.post('A tabulation timestep of <= 1 days is recommended.', 'WARN')
execute_string = execute_string + ' --polyephem-tabtimestep '+str(polyephem_tabtimestep)
casalog.post('Running ' + theexecutable
+ ' standalone invoked as:')
# print execute_string
casalog.post(execute_string)
exitcode = os.system(execute_string)
if exitcode != 0:
if not showversion:
casalog.post(theexecutable
+ ' terminated with exit code '
+ str(exitcode), 'SEVERE')
raise Exception, \
'ASDM conversion error. Please check if it is a valid ASDM and that data/alma/asdm is up to date.'
if showversion:
return
#
# Possibly remove the name of the measurement set expected to contain the corrected data from the list of of produced measurement
# sets if it appears the filler did not find any corrected data.
#
if not os.path.exists(visoc):
vistoproc = [myviso for myviso in vistoproc if myviso != visoc]
# CAS-7369. HISTORY should be written after createmms is tested
#
# Populate the HISTORY table of the MS with information about the context in which it's been created
#
try:
mslocal = mstool()
param_names = importasdm.func_code.co_varnames[:importasdm.func_code.co_argcount]
param_vals = [eval(p) for p in param_names]
for myviso in vistoproc:
write_history(mslocal, myviso, 'importasdm', param_names,
param_vals, casalog)
except Exception, instance:
casalog.post("*** Error \'%s\' updating HISTORY" % (instance),
'WARN')
return False
if mslocal:
mslocal = None
#
# Do we apply fixspwbackport
if remove_ref_undef :
casalog.post('remove_ref_undef=True: fixspwbackport will be applied ...')
for myviso in vistoproc:
cmd = 'fixspwbackport ' + myviso
casalog.post('Running fixspwbackport standalone invoked as:')
casalog.post(cmd)
cmdexitcode = os.system(cmd)
if cmdexitcode != 0:
casalog.post(cmd
+ ' terminated with exit code '
+ str(cmdexitcode), 'SEVERE')
raise Exception, 'fixspwbackport error.'
# Binary Flag processing
if bdfflags:
casalog.post('Parameter bdfflags==True: flags from the ASDM binary data will be used to set the MS flags ...')
bdffexecutable = 'bdflags2MS '
bdffexecstring_base = bdffexecutable + ' -f ALL' + ' --ocm "' + ocorr_mode \
+ '" --logfile "' + casalog.logfile() + '"'
if len(scans) > 0:
bdffexecstring_base = bdffexecstring_base + ' --scans ' + scans
if lazy and not compression:
bdffexecstring_base = bdffexecstring_base + ' --lazy=true'
for myviso in vistoproc:
if myviso.find("wvr-corrected") != -1:
options = " --wvr-corrected=True "
else:
options = " "
bdffexecstring = bdffexecstring_base + options + asdm + ' ' + myviso
casalog.post('Running '+bdffexecutable+' standalone invoked as:')
casalog.post(bdffexecstring)
bdffexitcode = os.system(bdffexecstring)
if bdffexitcode != 0:
casalog.post(bdffexecutable
+ ' terminated with exit code '
+ str(bdffexitcode), 'SEVERE')
raise Exception, \
'ASDM binary flags conversion error. Please check if it is a valid ASDM and that data/alma/asdm is up to date.'
theephemfields = ce.findattachedephemfields(myviso,field='*')
if len(theephemfields)>0:
# until asdm2MS does this internally: recalc the UVW coordinates for ephem fields
imt = imtool()
imt.open(myviso, usescratch=False)
imt.calcuvw(theephemfields, refcode='J2000', reuse=False)
imt.close()
if convert_ephem2geo:
for myviso in vistoproc:
ce.convert2geo(myviso, '*') # convert any attached ephemerides to GEO
if len(theephemfields)>0:
# also set the direction column in the SOURCE table
tblocal.open(myviso+'/FIELD', nomodify=False)
sourceids = tblocal.getcol('SOURCE_ID')
ftimes = tblocal.getcol('TIME')
ftimekw = tblocal.getcolkeywords('TIME')
tmpa = tblocal.getcol('PHASE_DIR')
origphasedir = tmpa
affectedsids = []
thesamplefields = []
for fld in theephemfields: # determine all source ids used by the ephem fields
if not (sourceids[fld] in affectedsids): # this source id wasn't handled yet
affectedsids.append(sourceids[fld])
thesamplefields.append(fld)
# need to temporarily change the offset (not all mosaics have an element at (0,0))
tmpa[0][0][fld]=0.
tmpa[1][0][fld]=0.
#endif
#endfor
tblocal.putcol('PHASE_DIR', tmpa)
tblocal.close()
tblocal.open(myviso+'/SOURCE')
sourceposref = tblocal.getcolkeywords('DIRECTION')['MEASINFO']['Ref']
tblocal.close()
directions = []
msmdlocal = casac.msmetadata()
msmdlocal.open(myviso)
for fld in thesamplefields:
thedirmeas = msmdlocal.phasecenter(fld)
if thedirmeas['refer']!=sourceposref:
casalog.post('Ephemeris is in '+thedirmeas['refer']+' instead of '+sourceposref
+' frame.\nEntry in SOURCE table will be converted to '+sourceposref, 'WARN')
melocal = metool()
melocal.doframe(thedirmeas)
thedirmeas = melocal.measure(thedirmeas, sourceposref)
directions.append([thedirmeas['m0']['value'], thedirmeas['m1']['value']])
thetime = me.epoch(v0=str(ftimes[fld])+'s', rf=ftimekw['MEASINFO']['Ref'])
casalog.post("Will set SOURCE direction for SOURCE_ID "+str(sourceids[fld])
+" to ephemeris phase center for time "+str(thetime['m0']['value'])+" "+thetime['m0']['unit']+" "+thetime['refer'])
#endfor
msmdlocal.close()
# restore original PHASE_DIR
tblocal.open(myviso+'/FIELD', nomodify=False)
tblocal.putcol('PHASE_DIR', origphasedir)
tblocal.close()
# write source directions
tblocal.open(myviso+'/SOURCE', nomodify=False)
ssourceids = tblocal.getcol('SOURCE_ID')
sdirs = tblocal.getcol('DIRECTION')
for row in xrange(0,len(ssourceids)):
for i in xrange(0,len(affectedsids)):
if ssourceids[row]==affectedsids[i]:
sdirs[0][row] = directions[i][0]
sdirs[1][row] = directions[i][1]
break
#endfor
#endfor
tblocal.putcol('DIRECTION', sdirs) # write back corrected directions
tblocal.close()
#end if
##############################################################################################3
# CAS-7369 - Create an output Multi-MS (MMS)
if createmms:
# Get the default parameters of partition
from tasks import partition
fpars = partition.parameters
for mypar in fpars.keys():
fpars[mypar] = partition.itsdefault(mypar)
# Call the cluster for each MS
for myviso in vistoproc:
casalog.origin('importasdm')
# Move original MS to tempdir
tempname = myviso+'.temp.ms'
outputmms = myviso
shutil.move(myviso, tempname)
# Get the proper column
datacolumn = 'DATA'
dcols = ['DATA', 'FLOAT_DATA']
for dc in dcols:
if len(th.getColDesc(tempname, dc)) > 0:
datacolumn = dc
break
fpars['datacolumn'] = datacolumn
casalog.post('Will create a Multi-MS for: '+myviso)
fpars['vis'] = tempname
fpars['flagbackup'] = False
fpars['outputvis'] = outputmms
fpars['separationaxis'] = separationaxis
fpars['numsubms'] = numsubms
pdh = ParallelDataHelper('partition', fpars)
# Get a cluster
pdh.setupCluster(thistask='partition')
try:
pdh.go()
# Remove original MS
shutil.rmtree(tempname)
except Exception, instance:
# Restore MS in case of error in MMS creation
shutil.move(tempname, myviso)
casalog.post('%s'%instance,'ERROR')
return False
casalog.origin('importasdm')
def tipopac(msname, caltableZ, tauPerAnt, calcTcals, caltableT, cmdFlag,
usrFlag, flagFile):
#
# Task tipopac
#
# Derive zenith opacity and Tcals from JVLA tip data.
# Christopher A. Hales
#
# Based on JIRA CASR-16, VLA Sci. Memo 170, and EVLA Memos 145 and 202.
# Originally written to support ngVLA Memo 63.
# See xml for code overview.
#
# Version 1.0 (tested with CASA Version 5.6.0 REL)
# 22 October 2019
casalog.origin('tipopac')
casalog.post('--> tipopac version 1.0')
# JVLA tips runs between approx 55-23 deg elevation (35-67 deg zenith angle, respectively)
porder = 3 # order of polynomial fit
zmin = 40. # zenith angle min, degrees
zmax = 62. # zenith angle max, degrees
# minimum number of (assuming 1 sec) integration times to get a good solution
# pick ~40 sec somewhat randomly. A normal tipping scan should take ~90 sec.
minTipInts = 40
# for calcTcals=True, set threshold warning level for % diff with TcalMS
Tdifthresh = 30
if calcTcals & tauPerAnt:
casalog.post(
"*** WARNING: Setting tauPerAnt=False because calcTcals=True.",
"WARN")
tauPerAnt = False
# avoid using global cb tool, which can cause issues with table cache
def gencaltableZ(msname, caltableZ):
mycb = casac.calibrater()
mycb.open(msname, False, False, False)
mycb.createcaltable(caltableZ, 'Real', 'TOpac', True)
mycb.close()
# temperature of resolved astrophysical background in noise K
def get_Trab(nu_Hz):
# Condon et al., 2012, ApJ, 758, 23
return 0.1 / (nu_Hz / 1.4e9)**(2.7)
# kinetic to noise temp in K
def k2nt(T, nu_Hz):
h = 6.6261e-34
k = 1.3806e-23
return T * (h * nu_Hz / (k * T) / (np.exp(h * nu_Hz / (k * T)) - 1.))
# temperature of unresolved astrophysical background in noise K
def get_Tuab(nu_Hz):
Tcmb = 2.725
return k2nt(Tcmb, nu_Hz)
# Tsys vs z tipping curve function
def func(z, params, Trab, Tuab, Twmtp):
# z in deg, T's all in noise K
# T0 = Tant + Trx1 + Trx2 + Tcal/2 ~= constant
T0, tau0 = params
Tsys = T0 + (Trab + Tuab)*np.exp(-tau0/np.cos(np.deg2rad(z))) + \
Twmtp*(1-np.exp(-tau0/np.cos(np.deg2rad(z))))
return Tsys
# option 1: calcTcals=False and tauPerAnt=True
# 3 unknown parameters: T0_pol0, T0_pol1, tau0
# option 2: calcTcals=False and tauPerAnt=False
# 2*nant+1 unknown parameters:
# T0_a0_p0, T0_a0_p1, ..., T0_aN_p0, T0_aN_p1, tau0
# option 3: same as 2, but Tsys values will be adjusted beforehand
# these can share the same wrapper (op1: N=1, op2/3: N=nant)
def err_multi_wrap(Trab, Tuab, Twmtp):
def err_multi(p, *argv):
N = len(argv) / 3
z = argv[:N]
Tsys = argv[N:]
params = p[0], p[-1]
errArr = Tsys[0] - func(z[0], params, Trab, Tuab, Twmtp)
for k in range(1, 2 * N):
params = p[k], p[-1]
errArr = np.concatenate([
errArr, Tsys[k] - func(z[k / 2], params, Trab, Tuab, Twmtp)
])
return errArr
return err_multi
### get antenna details
tb.open(msname + '/ANTENNA')
antNames = tb.getcol('NAME')
tb.close()
lenAnt = len(antNames)
### get full spw details for MS (not only tipping scans)
tb.open(msname + '/SPECTRAL_WINDOW')
spwCntFreq = np.mean(tb.getcol('CHAN_FREQ'), axis=0)
tb.close()
lenSpw = len(spwCntFreq)
### get pointing zenith angle
# check if pointing sub-table contains data. If not, give error and exit.
# this should be produced by importasdm in all cases (with_pointing_correction true or false)
casalog.post('--> Reading antenna pointing data.')
tb.open(msname + '/POINTING')
# read in elevation vs time, will include data from tips and also pointing if performed
# Note this is stored in ENCODER in AZELGEO coordinates
#tb.getcolkeywords('ENCODER')
#{'MEASINFO': {'Ref': 'AZELGEO', 'type': 'direction'},
# 'QuantumUnits': array(['rad', 'rad'], dtype='|S4')}
#
# CASA coordinate frames:
# https://casa.nrao.edu/casadocs/casa-5.1.0/reference-material/coordinate-frames
# extract pointing data for full observation
# not super efficient, but majority of data is expected to come from tipping scans.
# first, get max timestamps per antenna, in case they differ
maxAntT = 0
for a in range(lenAnt):
temptb = tb.query('ANTENNA_ID==' + str(a))
lenAntT = len(temptb.getcol('TIME'))
if lenAntT > maxAntT: maxAntT = lenAntT
if maxAntT == 0:
casalog.post("*** ERROR: That's strange, the pointing table is empty.",
"ERROR")
casalog.post("*** ERROR: Exiting tipopac.", "ERROR")
return
# 0 = time UTC seconds, 1 = zenith angle (deg)
dataPoint = np.zeros([lenAnt, maxAntT, 2])
me.doframe(me.observatory('VLA'))
for a in range(lenAnt):
casalog.post(' processing antenna ' + antNames[a] + ' (' +
str(a + 1) + '/' + str(lenAnt) + ')')
temptb = tb.query('ANTENNA_ID==' + str(a))
lenAntT = len(temptb.getcol('TIME'))
dataPoint[a, 0:lenAntT, 0] = temptb.getcol('TIME')
azel = temptb.getcol('ENCODER')
for i in range(lenAntT):
dataPoint[a, i, 1] = 90 - np.rad2deg(
me.measure(
me.direction('AZELGEO',
str(np.rad2deg(azel[0, i])) + 'deg',
str(np.rad2deg(azel[1, i])) + 'deg'),
'AZEL')['m1']['value'])
tb.close()
del temptb, azel
### read in time ranges for tipping scans and get associated spw's
casalog.post('--> Reading time ranges for tipping scans.')
msmd = casac.msmetadata()
msmd.open(msname)
# Only the scan with 2 subscans, with intent DO_SKYDIP, are of interest.
# The others can be ignored, they don't contain any useful data.
scans = msmd.scansforintent('*DO_SKYDIP*')
lenScans = len(scans)
tipSpw = msmd.spwsforintent('*DO_SKYDIP*')
lenTipSpw = len(tipSpw)
# get start and end time for each scan
times = np.zeros([lenScans, 2]) # UTC seconds
for i in range(lenScans):
times[i, 0] = msmd.timesforscan(scans[i])[0]
times[i, 1] = msmd.timesforscan(scans[i])[-1]
msmd.done()
### get estimated weighted mean atmospheric temperatures in kinetic temp K
casalog.post('--> Reading MS surface temperature data and '+\
'estimating weighted mean atmospheric temperatures.')
# sampled every approximately 1 minute
tb.open(msname + '/WEATHER')
tmp1 = tb.getcol('TIME')
# estimate weighted mean atmospheric temperature using
# Tm ~ 70.2 + 0.72 * Ts
# from Bevis et al., 1992, J. Geophys. Res. 97(D14), 15,787
tmp2 = tb.getcol('TEMPERATURE') * 0.72 + 70.2
tmp2[tb.getcol('TEMPERATURE_FLAG') == 1] = np.NaN
tb.close()
# time (UTC sec), temp (K)
dataTemp = np.column_stack((tmp1, tmp2))
del tmp1, tmp2
### read in online flags except for ANTENNA_NOT_ON_SOURCE
# subreflector errors shouldn't make any difference, but no harm flagging
if cmdFlag:
casalog.post('--> Reading online flags.')
tb.open(msname + '/FLAG_CMD')
if len(tb.getcol('REASON')) == 0:
casalog.post(
"*** ERROR: The online flag table (FLAG_CMD) is empty.",
"ERROR")
casalog.post("*** ERROR: Ensure that process_flags=True when running "+\
"importasdm.","ERROR")
casalog.post("*** ERROR: Exiting tipopac.", "ERROR")
return
temptb = tb.query("REASON!='ANTENNA_NOT_ON_SOURCE'")
dataCmdRaw = temptb.getcol('COMMAND')
lenDataCmd = len(dataCmdRaw)
# antenna, start time, end time (UTC sec)
dataCmd = np.zeros([lenDataCmd, 3])
for f in range(lenDataCmd):
dataCmd[f,0] = np.where(antNames==dataCmdRaw[f].\
replace("'","&&").split('&&')[1])[0][0]
dataCmd[f,1] = qa.quantity(dataCmdRaw[f].replace("'","&&").\
split('&&')[4].split('~')[0],'ymd')['value']*24*3600
dataCmd[f,2] = qa.quantity(dataCmdRaw[f].replace("'","&&").\
split('&&')[4].split('~')[1],'ymd')['value']*24*3600
tb.close()
### read in user-specified flags
if usrFlag:
casalog.post('--> Reading user-defined flags.')
dataUsrRaw = np.loadtxt(flagFile, dtype=str)
if dataUsrRaw.size == 3:
dataUsrRaw = dataUsrRaw.reshape([1, 3])
lenDataUsr = len(dataUsrRaw)
# antenna, spw, start time, end time (UTC sec)
dataUsr = np.zeros([lenDataUsr, 4])
mintime = times.min()
maxtime = times.max()
maxf = lenDataUsr
f = 0
for fraw in range(lenDataUsr):
allant = False
allspw = False
myant = dataUsrRaw[fraw, 0].replace("'", "").split('=')[1]
myspw = dataUsrRaw[fraw, 1].replace("'", "").split('=')[1]
mytime1 = qa.quantity(dataUsrRaw[fraw,2].replace("'","").\
split('=')[1].split('~')[0],'ymd')['value']*24*3600
mytime2 = qa.quantity(dataUsrRaw[fraw,2].replace("'","").\
split('=')[1].split('~')[1],'ymd')['value']*24*3600
if myant == '-1':
allant = True
else:
dataUsr[f, 0] = np.where(antNames == myant)[0][0]
if myspw == '-1':
allspw = True
else:
dataUsr[f, 1] = float(myspw)
dataUsr[f, 2] = mytime1
dataUsr[f, 3] = mytime2
# to avoid having to modify code below, copy flag command to all
# relevant ants, spws, times (not the most elegant solution...)
# only focus on following cases, within given time range:
# ant and spw specified
# ant specified with allspw
# allant and allspw
if (not allant) and (not allspw):
f += 1
elif (not allant) and (allspw):
# all spws for a given antenna in a given time range
dataUsr = np.insert(dataUsr,
f,
np.zeros([lenSpw - 1, 4]),
axis=0)
for s in range(lenSpw):
dataUsr[f, 0] = np.where(antNames == myant)[0][0]
dataUsr[f, 1] = s
dataUsr[f, 2] = mytime1
dataUsr[f, 3] = mytime2
f += 1
maxf += lenSpw - 1
elif (allant) and (allspw):
# all antennas and all spws in a given time range
dataUsr = np.insert(dataUsr,
f,
np.zeros([lenAnt * lenSpw - 1, 4]),
axis=0)
for a in range(lenAnt):
for s in range(lenSpw):
dataUsr[f, 0] = a
dataUsr[f, 1] = s
dataUsr[f, 2] = mytime1
dataUsr[f, 3] = mytime2
f += 1
maxf += lenAnt * lenSpw - 1
else:
casalog.post("*** ERROR: Manual flags not specified "+\
"according to instructions in help.","ERROR")
casalog.post("*** ERROR: Exiting tipopac.", "ERROR")
return
### read in switched power psum and pdif per tip scan and apply flags
casalog.post('--> Reading switched power data.')
# start by reading in stored Tcals (K)
tb.open(msname + '/CALDEVICE')
# antenna, spw, pol
dataTcalMS = np.zeros([lenAnt, lenSpw, 2])
for a in range(lenAnt):
for s in range(lenSpw):
temptb = tb.query('ANTENNA_ID==' + str(a) +
'&&SPECTRAL_WINDOW_ID==' + str(s))
# rows 0/1 = noise tube/solar filter
# cols 0/1 = R/L
dataTcalMS[a, s, 0] = temptb.getcol('NOISE_CAL')[0, 0, 0]
dataTcalMS[a, s, 1] = temptb.getcol('NOISE_CAL')[0, 1, 0]
tb.close()
# create Z caltable to be filled in below
gencaltableZ(msname, caltableZ)
# set default values with everything flagged
caltableNrows = lenScans * lenSpw * lenAnt
tb.open(caltableZ, nomodify=False)
tb.addrows(caltableNrows)
k = 0
for i in range(lenScans):
for a in range(lenAnt):
for s in range(lenSpw):
tb.putcell('TIME', k, (times[i, 0] + times[i, 1]) / 2.)
tb.putcell('FIELD_ID', k, -1)
tb.putcell('SPECTRAL_WINDOW_ID', k, s)
tb.putcell('ANTENNA1', k, a)
tb.putcell('ANTENNA2', k, -1)
tb.putcell('SCAN_NUMBER', k, i)
tb.putcell('FPARAM', k, np.array([[0.]]))
tb.putcell('PARAMERR', k, np.array([[0.]]))
tb.putcell('FLAG', k, np.array([[True]], dtype=bool))
tb.putcell('SNR', k, np.array([[1.]]))
# the WEIGHT column can be left with empty cells
k += 1
tb.flush()
tb.close()
if calcTcals:
tb.open(msname + '/CALDEVICE')
newtab = tb.copy(caltableT,
deep=True,
valuecopy=True,
norows=True,
returnobject=True)
tb.close()
newtab.close()
tb.open(caltableT, nomodify=False)
tb.addrows(caltableNrows)
k = 0
for i in range(lenScans):
for a in range(lenAnt):
for s in range(lenSpw):
tb.putcell('ANTENNA_ID', k, a)
tb.putcell('SPECTRAL_WINDOW_ID', k, s)
tb.putcell('TIME', k, (times[i, 0] + times[i, 1]) / 2.)
tb.putcell('NUM_CAL_LOAD', k, 2)
tb.putcell(
'CAL_LOAD_NAMES', k,
np.array([['NOISE_TUBE_LOAD'], ['SOLAR_FILTER']]))
tb.putcell('NUM_RECEPTOR', k, 2)
tb.putcell('NOISE_CAL', k, np.array([[0., 0.], [0., 0.]]))
# other columns can default
k += 1
tb.flush()
tb.close()
# first, get all data wrt zenith angle into a master array
# JVLA tipping scans don't run for more than 2 mins with 1 sec sampling
# scan, ant, spw, pol, timestamp: 0=ZA[deg], 1=Twmt[kinetic K], 2=Tsys'[K]
# if calcTcals: 3=delta(Tsys') between z_min and z_max from polynomial fit,
# this isn't very efficient; meh, the runtime is dominated by getcol, and data1 size isn't huge
# (ZA and Twmt are not spw or pol dependent ... indeed Twmt isn't even antenna dependent, meh2)
# (and Twmt will be effectively time-independent during a scan, meh3)
if calcTcals:
data1 = np.zeros([lenScans, lenAnt, lenSpw, 2, 120, 3 + 1])
else:
data1 = np.zeros([lenScans, lenAnt, lenSpw, 2, 120, 3])
tb.open(msname + '/SYSPOWER')
for i in range(lenScans):
casalog.post('--> Gathering data for scan ' + str(scans[i]) + ' (' +
str(i + 1) + '/' + str(lenScans) + ')')
casalog.filter('WARN')
msmd.open(msname)
scanspws = msmd.spwsforscan(scans[i])
lenScanSpws = len(scanspws)
msmd.done()
casalog.filter('INFO')
for a in range(lenAnt):
casalog.post(' antenna ' + antNames[a] + ' (' + str(a + 1) +
'/' + str(lenAnt) + ')')
for s in scanspws:
casalog.post(' spw ' + str(s) + ' (' +
str(s - scanspws[0] + 1) + '/' +
str(lenScanSpws) + ')')
subtb = tb.query('TIME>='+str(times[i,0])+'&&TIME<='+str(times[i,1])+\
'&&ANTENNA_ID=='+str(a)+'&&SPECTRAL_WINDOW_ID=='+str(s))
spT = subtb.getcol('TIME')
lenSpT = len(spT)
if lenSpT > 0:
# cols 0/1 = R/L
pdif = subtb.getcol('SWITCHED_DIFF')
psum = subtb.getcol('SWITCHED_SUM')
#rq = subtb.getcol('REQUANTIZER_GAIN')
# rq not needed. Pdif reported in MS appears to neglect digital gain factor
# apply online flags except for ANTENNA_NOT_ON_SOURCE
# there is probably a smarter way to do this...
# cmd case 1: flagging starts and ends within tip
tmp = dataCmd[np.where((dataCmd[:,0]==a) &\
(dataCmd[:,1]>=spT[0]) &\
(dataCmd[:,2]<=spT[-1]))]
for f in range(len(tmp)):
pdif[:,
np.where((spT >= tmp[f, 1])
& (spT <= tmp[f, 2]))] = np.NaN
psum[:,
np.where((spT >= tmp[f, 1])
& (spT <= tmp[f, 2]))] = np.NaN
# cmd case 2: flagging starts before tip and ends within tip
tmp = dataCmd[np.where((dataCmd[:,0]==a) &\
(dataCmd[:,1]<=spT[0]) &\
(dataCmd[:,2]>=spT[0]) &\
(dataCmd[:,2]<=spT[-1]))]
for f in range(len(tmp)):
pdif[:,
np.where((spT >= tmp[f, 1])
& (spT <= tmp[f, 2]))] = np.NaN
psum[:,
np.where((spT >= tmp[f, 1])
& (spT <= tmp[f, 2]))] = np.NaN
# cmd case 3: flagging starts during tip and ends after tip
tmp = dataCmd[np.where((dataCmd[:,0]==a) &\
(dataCmd[:,1]>=spT[0]) &\
(dataCmd[:,1]<=spT[-1]) &\
(dataCmd[:,2]>=spT[-1]))]
for f in range(len(tmp)):
pdif[:,
np.where((spT >= tmp[f, 1])
& (spT <= tmp[f, 2]))] = np.NaN
psum[:,
np.where((spT >= tmp[f, 1])
& (spT <= tmp[f, 2]))] = np.NaN
# cmd case 4: flagging starts before tip and ends after tip
tmp = dataCmd[np.where((dataCmd[:,0]==a) &\
(dataCmd[:,1]<=spT[0]) &\
(dataCmd[:,2]>=spT[-1]))]
for f in range(len(tmp)):
pdif[:,
np.where((spT >= tmp[f, 1])
& (spT <= tmp[f, 2]))] = np.NaN
psum[:,
np.where((spT >= tmp[f, 1])
& (spT <= tmp[f, 2]))] = np.NaN
# apply manual flags
# usr case 1: flagging starts and ends within tip
tmp = dataUsr[np.where((dataUsr[:,0]==a) &\
(dataUsr[:,1]==s) &\
(dataUsr[:,2]>=spT[0]) &\
(dataUsr[:,3]<=spT[-1]))]
for f in range(len(tmp)):
pdif[:,
np.where((spT >= tmp[f, 2])
& (spT <= tmp[f, 3]))] = np.NaN
psum[:,
np.where((spT >= tmp[f, 2])
& (spT <= tmp[f, 3]))] = np.NaN
# usr case 2: flagging starts before tip and ends within tip
tmp = dataUsr[np.where((dataUsr[:,0]==a) &\
(dataUsr[:,1]==s) &\
(dataUsr[:,2]<=spT[0]) &\
(dataUsr[:,3]>=spT[0]) &\
(dataUsr[:,3]<=spT[-1]))]
for f in range(len(tmp)):
pdif[:,
np.where((spT >= tmp[f, 2])
& (spT <= tmp[f, 3]))] = np.NaN
psum[:,
np.where((spT >= tmp[f, 2])
& (spT <= tmp[f, 3]))] = np.NaN
# usr case 3: flagging starts during tip and ends after tip
tmp = dataUsr[np.where((dataUsr[:,0]==a) &\
(dataUsr[:,1]==s) &\
(dataUsr[:,2]>=spT[0]) &\
(dataUsr[:,2]<=spT[-1]) &\
(dataUsr[:,3]>=spT[-1]))]
for f in range(len(tmp)):
pdif[:,
np.where((spT >= tmp[f, 2])
& (spT <= tmp[f, 3]))] = np.NaN
psum[:,
np.where((spT >= tmp[f, 2])
& (spT <= tmp[f, 3]))] = np.NaN
# usr case 4: flagging starts before tip and ends after tip
tmp = dataUsr[np.where((dataUsr[:,0]==a) &\
(dataUsr[:,1]==s) &\
(dataUsr[:,2]<=spT[0]) &\
(dataUsr[:,3]>=spT[-1]))]
for f in range(len(tmp)):
pdif[:,
np.where((spT >= tmp[f, 2])
& (spT <= tmp[f, 3]))] = np.NaN
psum[:,
np.where((spT >= tmp[f, 2])
& (spT <= tmp[f, 3]))] = np.NaN
del tmp
# put the following info into data1 (for each poln, meh)
for x in range(lenSpT):
# get pointing zenith angle in deg at spT times
# pointing data for the JVLA is recorded every approximately 0.1 sec
# don't bother interpolating to swpow timestamps, which for
# the JVLA are recorded every 1 sec. Just take nearest value.
data1[i, a, s, :, x,
0] = dataPoint[a, (np.abs(dataPoint[a, :, 0] -
spT[x])).argmin(), 1]
# get Tatm in kinetic K at spT times
# MS Tsurf is only sampled approximately every minute
# So it's perhaps worth interpolating temperatures at the switched power timestamps
data1[i, a, s, :, x,
1] = np.interp(spT[x], dataTemp[:, 0],
dataTemp[:, 1])
for p in range(2):
# calculate Tsys = (Psum/2)/Pdif * Tcal
data1[i, a, s, p, 0:lenSpT,
2] = (psum[p] /
2.) / (pdif[p]) * dataTcalMS[a, s, p]
# flag Tsys if pdif<0 or psum<0
tmpIndx = np.where(pdif[p] < 0)[0]
data1[i, a, s, p, tmpIndx, 2] = np.NaN
tmpIndx = np.where(psum[p] < 0)[0]
data1[i, a, s, p, tmpIndx, 2] = np.NaN
if len(np.where(
data1[i, a, s, p, :, 2] > 0)[0]) < minTipInts:
data1[i, a, s, p, :, 2] = np.NaN
if p == 0:
polstr = 'R'
else:
polstr = 'L'
casalog.post('*** WARNING: '+antNames[a]+' spw '+str(s)+' poln '+polstr+\
' completely flagged in scan '+str(scans[i])+' due to insufficient unflagged'+\
' data after manual flagging or abnormal negative switched power data.','WARN')
if calcTcals:
indx = np.where(data1[i, a, s, 0, :, 2] > 0)[0]
if len(indx) > 0:
funcp = np.poly1d(
np.ma.polyfit(data1[i, a, s, p, indx, 0],
data1[i, a, s, p, indx,
2], porder))
data1[i, a, s, p, :,
3] = funcp(zmax) - funcp(zmin)
#from matplotlib import pyplot as plt
#i=0;a=20;s=19
#indx = np.where(data1[i,a,s,0,:,2]>0)[0]
#funcp = np.poly1d(np.ma.polyfit(data1[i,a,s,p,indx,0],data1[i,a,s,p,indx,2],porder))
#plt.plot(data1[i,a,s,p,indx,0],data1[i,a,s,p,indx,2],'b.-',data1[i,a,s,p,indx,0],funcp(data1[i,a,s,p,indx,0]),'r.-')
tb.close()
## proceed with nominated solution type
casalog.post(
'--> Calculating opacities and system temperature contributions.')
if calcTcals:
casalog.post(
' Zenith opacities (tau0), ant+elec contributions (Tae=Tant+Trx1+Trx2), and Tcal_new with % change from Tcal_MS are reported below.'
)
casalog.post(
' Results will be highlighted if the abs(change) in Tcal_new for R or L is >= '
+ '{:.0f}'.format(Tdifthresh) +
'%. Check Tcal solutions carefully.')
else:
casalog.post(
' Zenith opacities (tau0) and ant+elec contributions (Tae=Tant+Trx1+Trx2) are reported below.'
)
dataTae = np.zeros([lenScans, lenAnt, lenSpw, 2])
if (not calcTcals) and (tauPerAnt):
#
# OPTION 1: solve for opacity per scan, antenna, and spw (combined solve over both polarizations)
#
tb.open(caltableZ, nomodify=False)
dataopZ = np.zeros([lenScans, lenAnt, lenSpw])
for i in range(lenScans):
#casalog.post('--> Processing scan '+str(scans[i])+' ('+str(i+1)+'/'+str(lenScans)+')')
for a in range(lenAnt):
#casalog.post(' processing antenna '+antNames[a]+' ('+str(a+1)+'/'+str(lenAnt)+')')
for s in range(lenSpw):
#casalog.post(' processing spectral window '+str(s)+' ('+str(s+1)+'/'+str(lenSpw)+')')
# can expect that flagging will be polarization independent
# Tsys in data1 can be 0 (dummy value in array) or flagged (NaN)
# only process if valid Tsys solutions are available in, say, pol=0
indx = np.where(data1[i, a, s, 0, :, 2] > 0)[0]
if len(indx) > 0:
# for this scan, ant, spw: we have 2 datasets (Tsys vs ZA for 2 pols)
# and the equations have 3 unknowns (T0_pol1, T0_pol2, tau0)
Trab = get_Trab(spwCntFreq[s])
Tuab = get_Tuab(spwCntFreq[s])
# convert kinetic Twmt to noise temp in K
# hmm, to simplify, take mean weighted mean atmospheric temperature during scan
Twmtp = k2nt(np.mean(data1[i, a, s, 0, indx, 1]),
spwCntFreq[s])
# starting estimate for unknown parameters (T0_pol1, T0_pol2, tau0)
se = [50., 50., 0.2]
fit, ier = scipy.optimize.leastsq(
err_multi_wrap(Trab, Tuab, Twmtp),
se,
args=(data1[i, a, s, 0, indx,
0], data1[i, a, s, 0, indx,
2], data1[i, a, s, 1, indx,
2]))
#print fit
#x = data1[i,a,s,0,indx,0]
#y1 = data1[i,a,s,0,indx,2]
#y2 = data1[i,a,s,1,indx,2]
#from matplotlib import pyplot as plt
#plt.plot(x,y1,'b.-',x,func(x,np.r_[fit[0],fit[-1]],Trab,Tuab,Twmtp),'r.-',
# x,y2,'g.-',x,func(x,np.r_[fit[1],fit[-1]],Trab,Tuab,Twmtp),'y.-')
#plt.show()
dataopZ[i, a, s] = fit[-1]
dataTae[i, a, s, 0] = fit[0] - dataTcalMS[a, s, 0] / 2.
dataTae[i, a, s, 1] = fit[1] - dataTcalMS[a, s, 1] / 2.
casalog.post(' scan '+str(scans[i])+', '+antNames[a]+', spw '+str(s)+\
' - tau0: '+'{:.3f}'.format(fit[-1])+', Tae (K): '+\
'{:.2f}'.format(dataTae[i,a,s,0])+' (R), '+\
'{:.2f}'.format(dataTae[i,a,s,1])+' (L)')
k = i * lenAnt * lenSpw + a * lenSpw + s
tb.putcell('FPARAM', k, np.array([[fit[-1]]]))
tb.putcell('FLAG', k, np.array([[False]], dtype=bool))
tb.flush()
tb.close()
else:
#
# OPTION 2: solve for opacity per scan and spw (combined solve over all antennas and polarizations)
# OPTION 3: same as option 2, but also solve for Tcals
#
dataopZ = np.zeros([lenScans, lenSpw])
if calcTcals:
# 0=R, 1=L, 2=%difference R, 3=%difference L
# diff = (new-old)/old*100
dataTcal = np.zeros([lenScans, lenAnt, lenSpw, 4])
for i in range(lenScans):
casalog.filter('WARN')
msmd.open(msname)
scanspws = msmd.spwsforscan(scans[i])
msmd.done()
casalog.filter('INFO')
for s in scanspws:
# store data in prep for fitting
dataZA = ()
dataTsys = ()
se = []
AntArr = []
getTruw = True
if calcTcals:
dTsysp_median = np.nanmedian(data1[i, :, s, :, 0, 3])
for a in range(lenAnt):
indx = np.where(data1[i, a, s, 0, :, 2] > 0)[0]
if len(indx) > 0:
# only need to get Trab,Tuab,Twmtp once, same for all antennas
# Unlikely that temperature changes much over 2 mins, so
# don't worry about potential flagging differences between ants
if getTruw:
Trab = get_Trab(spwCntFreq[s])
Tuab = get_Tuab(spwCntFreq[s])
Twmtp = k2nt(np.mean(data1[i, a, s, 0, indx, 1]),
spwCntFreq[s])
getTruw = False
if calcTcals:
for p in range(2):
C = dTsysp_median / data1[i, a, s, p, 0, 3]
dataTsys += (C * data1[i, a, s, p, indx, 2], )
dataTcal[i, a, s, p] = C * dataTcalMS[a, s, p]
dataTcal[i, a, s, p + 2] = (C - 1) * 100.
else:
dataTsys += (
data1[i, a, s, 0, indx, 2],
data1[i, a, s, 1, indx, 2],
)
dataZA += (data1[i, a, s, 0, indx, 0], )
se += [50., 50.]
AntArr += [a]
se += [0.2]
fit, ier = scipy.optimize.leastsq(err_multi_wrap(
Trab, Tuab, Twmtp),
se,
args=dataZA + dataTsys)
#import matplotlib.pyplot as plt
#a=4
#p=0; plt.plot(dataZA[a],dataTsys[2*a+p],'b.-',dataZA[a],func(dataZA[a],[fit[2*a+p],fit[-1]],Trab,Tuab,Twmtp),'r-')
#p=1; plt.plot(dataZA[a],dataTsys[2*a+p],'b.-',dataZA[a],func(dataZA[a],[fit[2*a+p],fit[-1]],Trab,Tuab,Twmtp),'r-')
dataopZ[i, s] = fit[-1]
m = 0
for a in AntArr:
logpriority = 'INFO'
if calcTcals:
dataTae[i, a, s,
0] = fit[2 * m] - dataTcal[i, a, s, 0] / 2.
dataTae[i, a, s,
1] = fit[2 * m + 1] - dataTcal[i, a, s, 1] / 2.
extrastr = ', Tcal_new (K): '+\
'{:.2f}'.format(dataTcal[i,a,s,0])+' (R), '+\
'{:.2f}'.format(dataTcal[i,a,s,1])+' (L), '+\
'% change from Tcal_MS: '+\
'{:.1f}'.format(dataTcal[i,a,s,2])+' (R), '+\
'{:.1f}'.format(dataTcal[i,a,s,3])+' (L)'
if (np.abs(dataTcal[i, a, s, 2]) >= Tdifthresh) or (
np.abs(dataTcal[i, a, s, 3]) >= Tdifthresh):
logpriority = 'WARN'
else:
dataTae[i, a, s,
0] = fit[2 * m] - dataTcalMS[a, s, 0] / 2.
dataTae[i, a, s,
1] = fit[2 * m + 1] - dataTcalMS[a, s, 1] / 2.
extrastr = ''
casalog.post(' scan '+str(scans[i])+', spw '+str(s)+', '+antNames[a]+\
' - tau0: '+'{:.3f}'.format(fit[-1])+', Tae (K): '+\
'{:.2f}'.format(dataTae[i,a,s,0])+' (R), '+\
'{:.2f}'.format(dataTae[i,a,s,1])+' (L)'+extrastr,logpriority)
m += 1
k = i * lenAnt * lenSpw + a * lenSpw + s
tb.open(caltableZ, nomodify=False)
tb.putcell('FPARAM', k, np.array([[fit[-1]]]))
tb.putcell('FLAG', k, np.array([[False]], dtype=bool))
tb.flush()
tb.close()
if calcTcals:
tb.open(caltableT, nomodify=False)
tb.putcell(
'NOISE_CAL', k,
np.array(
[[dataTcal[i, a, s, 0], dataTcal[i, a, s, 1]],
[0., 0.]]))
tb.flush()
tb.close()
# print out summary statistics
casalog.post(
'--> Print summary statistics for zenith opacity (over antenna) in nepers: '
)
if (not calcTcals):
if tauPerAnt:
#
# OPTION 1: opacity was solved per scan, antenna, and spw
#
casalog.post(
' median, median absolute deviation, min outlier, max outlier.'
)
for i in range(lenScans):
casalog.post(' scan ' + str(scans[i]) + ':')
casalog.filter('WARN')
msmd.open(msname)
scanspws = msmd.spwsforscan(scans[i])
msmd.done()
casalog.filter('INFO')
for s in scanspws:
# value of zero could be present if all data was flagged
# don't let this contribute to statistics
sA = dataopZ[i, :, s]
sB = sA[np.abs(dataopZ[i, :, s]) > 0]
s1 = np.median(sB)
s2 = np.median(np.abs(sB - s1))
s3 = np.min(sB)
s4 = np.max(sB)
casalog.post(' spw '+str(s)+' ({:.4f} GHz): '.format(spwCntFreq[s]/1e9)+\
'{:6.3f}'.format(s1)+', '+'{:6.3f}'.format(s2)+', '+\
'{:6.3f}'.format(s3)+', '+'{:6.3f}'.format(s4))
else:
#
# OPTION 2: opacity was solved per scan and spw
#
for i in range(lenScans):
casalog.post(' scan ' + str(scans[i]) + ':')
casalog.filter('WARN')
msmd.open(msname)
scanspws = msmd.spwsforscan(scans[i])
msmd.done()
casalog.filter('INFO')
for s in scanspws:
casalog.post(' spw '+str(s)+' ({:.4f} GHz): '.format(spwCntFreq[s]/1e9)+\
'{:6.3f}'.format(dataopZ[i,s]))
else:
#
# OPTION 3: same as option 2
#
for i in range(lenScans):
casalog.post(' scan ' + str(scans[i]) + ':')
casalog.filter('WARN')
msmd.open(msname)
scanspws = msmd.spwsforscan(scans[i])
msmd.done()
casalog.filter('INFO')
for s in scanspws:
casalog.post(' spw '+str(s)+' ({:.4f} GHz): '.format(spwCntFreq[s]/1e9)+\
'{:6.3f}'.format(dataopZ[i,s]))
casalog.post(
'--> Print summary statistics for Tae (over antenna and polarization) in K: '
)
casalog.post(
' median, median absolute deviation, min outlier, max outlier')
for i in range(lenScans):
casalog.post(' scan ' + str(scans[i]) + ':')
casalog.filter('WARN')
msmd.open(msname)
scanspws = msmd.spwsforscan(scans[i])
msmd.done()
casalog.filter('INFO')
for s in scanspws:
# value of zero could be present if all data was flagged
# don't let this contribute to statistics
sA = dataTae[i, :, s, :]
sB = sA[np.abs(dataTae[i, :, s, :]) > 0]
s1 = np.median(sB)
s2 = np.median(np.abs(sB - s1))
s3 = np.min(sB)
s4 = np.max(sB)
casalog.post(' spw '+str(s)+' ({:.4f} GHz): '.format(spwCntFreq[s]/1e9)+\
'{:8.3f}'.format(s1)+', '+'{:8.3f}'.format(s2)+', '+\
'{:8.3f}'.format(s3)+', '+'{:8.3f}'.format(s4))
if calcTcals:
casalog.post(
'--> Print summary statistics for new Tcal solutions (over antenna and polarization) in K: '
)
casalog.post(
' median, median absolute deviation, min outlier, max outlier')
casalog.post(' scan ' + str(scans[i]) + ':')
casalog.filter('WARN')
msmd.open(msname)
scanspws = msmd.spwsforscan(scans[i])
msmd.done()
casalog.filter('INFO')
for s in scanspws:
# value of zero could be present if all data was flagged
# don't let this contribute to statistics
sA = dataTcal[i, :, s, 0:2]
sB = sA[np.abs(dataTcal[i, :, s, 0:2]) > 0]
s1 = np.median(sB)
s2 = np.median(np.abs(sB - s1))
s3 = np.min(sB)
s4 = np.max(sB)
casalog.post(' spw '+str(s)+' ({:.4f} GHz): '.format(spwCntFreq[s]/1e9)+\
'{:8.3f}'.format(s1)+', '+'{:8.3f}'.format(s2)+', '+\
'{:8.3f}'.format(s3)+', '+'{:8.3f}'.format(s4))
casalog.post(
'--> Print summary statistics for dTcal(%) = (Tcal_new-Tcal_ref)/Tcal_ref*100 (over antenna and polarization): '
)
casalog.post(
' median, median absolute deviation, min outlier, max outlier')
casalog.post(' scan ' + str(scans[i]) + ':')
casalog.filter('WARN')
msmd.open(msname)
scanspws = msmd.spwsforscan(scans[i])
msmd.done()
casalog.filter('INFO')
for s in scanspws:
# value of zero could be present if all data was flagged
# don't let this contribute to statistics
sA = dataTcal[i, :, s, 2:4]
sB = sA[np.abs(dataTcal[i, :, s, 2:4]) > 0]
s1 = np.median(sB)
s2 = np.median(np.abs(sB - s1))
s3 = np.min(sB)
s4 = np.max(sB)
casalog.post(' spw '+str(s)+' ({:.4f} GHz): '.format(spwCntFreq[s]/1e9)+\
'{:8.3f}'.format(s1)+', '+'{:8.3f}'.format(s2)+', '+\
'{:8.3f}'.format(s3)+', '+'{:8.3f}'.format(s4))