def sdsmooth(infile=None, datacolumn=None, antenna=None,
field=None, spw=None, timerange=None, scan=None,
pol=None, intent=None, reindex=None,
kernel=None, kwidth=None,
outfile=None, overwrite=None):
casalog.origin('sdsmooth')
try:
if len(outfile) == 0:
errmsg = 'outfile is empty.'
raise_exception(errmsg)
if (os.path.exists(outfile)) and (not overwrite):
errmsg = outfile+' exists.'
raise_exception(errmsg)
sdms.open(infile)
sdms.set_selection(spw=spw, field=field,
antenna=antenna,
timerange=timerange, scan=scan,
polarization=pol, intent=intent,
reindex=reindex)
sdms.smooth(type=kernel, width=kwidth, datacolumn=datacolumn, outfile=outfile)
# Write to HISTORY of outfile MS
param_names = sdsmooth.func_code.co_varnames[:sdsmooth.func_code.co_argcount]
param_vals = [eval(p) for p in param_names]
write_history(ms, outfile, 'sdsmooth', param_names,
param_vals, casalog)
except Exception, instance:
raise Exception, instance
def clearcal(
vis=None,
field=None,
spw=None,
intent=None,
addmodel=None,
):
casalog.origin('clearcal')
# Do the trivial parallelization
if ParallelTaskHelper.isParallelMS(vis):
helper = ParallelTaskHelper('clearcal', locals())
helper.go()
return
# Local versions of the tools
tblocal = tbtool()
cblocal = cbtool()
mslocal = mstool()
try:
# we will initialize scr cols only if we don't create them
doinit = False
if (type(vis) == str) & os.path.exists(vis):
tblocal.open(vis)
doinit = tblocal.colnames().count('CORRECTED_DATA') > 0
tblocal.close()
# We ignore selection if creating the scratch columns
if not doinit:
casalog.post('Need to create scratch columns; ignoring selection.'
)
cblocal.setvi(old=True,quiet=False); # Old VI for now
cblocal.open(vis, addmodel=addmodel)
else:
raise Exception, \
'Visibility data set not found - please verify the name'
# If necessary (scr col not just created), initialize scr cols
if doinit:
cblocal.selectvis(field=field, spw=spw, intent=intent)
cblocal.initcalset(1)
cblocal.close()
# Write history to the MS
param_names = clearcal.func_code.co_varnames[:clearcal.func_code.co_argcount]
param_vals = [eval(p) for p in param_names]
casalog.post('Updating the history in the output', 'DEBUG1')
write_history(mslocal, vis, 'clearcal', param_names,
param_vals, casalog)
except Exception, instance:
print '*** Error ***', instance
def importuvfits(fitsfile, vis, antnamescheme=None):
"""
Convert a UVFITS file to a CASA visibility data set (MS):
Keyword arguments:
fitsfile -- Name of input UV FITS file
default = none; example='3C273XC1.fits'
vis -- Name of output visibility file (MS)
default = none; example: vis='3C273XC1.ms'
antnamescheme -- Naming scheme for VLA/JVLA/CARMA antennas
default = old;
old: Antenna name is a number, '04'
This option exists for backwards compatibility
but can lead to ambiguous results when antenna
indices are used for data selection.
new: Antenna name is not a number, 'VA04' or 'EA04'
With this scheme, data selection via
antenna names and indices is non-ambiguous.
async -- Run asynchronously
default = false; do not run asychronously
"""
myms = mstool()
try:
try:
casalog.origin('importuvfits')
casalog.post("")
myms.fromfits(vis, fitsfile, antnamescheme=antnamescheme)
myms.close()
except Exception, instance:
casalog.post( str( '*** Error ***') + str(instance), 'SEVERE')
raise
# Write the args to HISTORY.
try:
param_names = \
importuvfits.func_code.co_varnames[:importuvfits.func_code.co_argcount]
param_vals = [eval(p) for p in param_names]
write_history(
myms, vis, 'importuvfits', param_names,
param_vals, casalog
)
except Exception, instance:
casalog.post("Failed to updated HISTORY table", 'WARN')
def importnro(infile=None, outputvis=None, overwrite=None, parallel=None):
"""
"""
casalog.origin('importnro')
status = True
try:
outputvis_temp = outputvis + '-backup-' + datetime.datetime.now(
).strftime('%Y%m%d-%H%M%S')
if os.path.exists(outputvis):
if overwrite:
os.rename(outputvis, outputvis_temp)
else:
raise RuntimeError('%s exists.' % (outputvis))
if not is_nostar(infile):
raise RuntimeError('%s is not a valid NOSTAR data.' % (infile))
status = mysdms.importnro(infile, outputvis, parallel)
if status:
# initialize weights using cb tool
mycb.open(outputvis, compress=False, addcorr=False, addmodel=False)
mycb.initweights(wtmode='nyq')
if os.path.exists(outputvis_temp):
shutil.rmtree(outputvis_temp)
else:
if os.path.exists(outputvis):
shutil.rmtree(outputvis)
if os.path.exists(outputvis_temp):
os.rename(outputvis_temp, outputvis)
raise RuntimeError('import failed.')
# Write parameters to HISTORY table of MS
param_names = importnro.func_code.co_varnames[:importnro.func_code.
co_argcount]
param_vals = [eval(p) for p in param_names]
write_history(myms, outputvis, 'importnro', param_names, param_vals,
casalog)
return status
except Exception, instance:
casalog.post('*** Error *** ' + str(instance), 'SEVERE')
raise instance
def uvsub(vis=None,reverse=False):
"""Subtract model from the corrected visibility data
uvsub(vis='ngc5921.ms', reverse=false)
This function subtracts model visibility data from corrected visibility
data leaving the residuals in the corrected data column. If the
parameter 'reverse' is set true, the process is reversed.
Keyword arguments:
vis -- Name of input visibility file (MS)
default: none; example: vis='ngc5921.ms'
reverse -- Reverse the operation (add rather than subtract)
default: false; example: reverse=true
uvsub(vis='ngc5921.ms', reverse=false)
"""
#Python script
#
try:
ms = mstool()
casalog.origin('uvsub')
if ((type(vis)==str) & (os.path.exists(vis))):
ms.open(thems=vis,nomodify=False)
else:
raise Exception, 'Visibility data set not found - please verify the name'
return
ms.uvsub(reverse)
ms.close
# Write history to MS
try:
param_names = uvsub.func_code.co_varnames[:uvsub.func_code.co_argcount]
param_vals = [eval(p) for p in param_names]
write_history(mstool(), vis, 'uvsub', param_names,
param_vals, casalog)
except Exception, instance:
casalog.post("*** Error \'%s\' updating HISTORY" % (instance),
'WARN')
return
def initweights(vis=None,wtmode=None,tsystable=None,gainfield=None,interp=None,spwmap=None,dowtsp=None):
casalog.origin('initweights')
# Do the trivial parallelization
if ParallelTaskHelper.isMPIEnabled() and ParallelTaskHelper.isParallelMS(vis):
tsystable = ParallelTaskHelper.findAbsPath(tsystable)
helper = ParallelTaskHelper('initweights', locals())
helper.go()
# Write history to MS.
try:
param_names = initweights.func_code.co_varnames[:initweights.func_code.co_argcount]
param_vals = [eval(p) for p in param_names]
casalog.post('Updating the history in the output', 'DEBUG1')
write_history(ms, vis, 'initweights', param_names,
param_vals, casalog)
except Exception, instance:
casalog.post("*** Error \'%s\' updating HISTORY" % (instance),
'WARN')
return
def delmod(vis=None, otf=None, field=None, scr=None):
casalog.origin('delmod')
# Do the trivial parallelization
if ParallelTaskHelper.isParallelMS(vis):
helper = ParallelTaskHelper('delmod', locals())
helper.go()
return
#Python script
try:
# only if vis exists...
if ((type(vis) == str) & (os.path.exists(vis))):
# ... and we are asked to do something...
# open without adding anything!
_cb.open(vis, addcorr=False, addmodel=False)
_cb.delmod(otf=otf, field=field, scr=scr)
_cb.close()
else:
raise Exception, 'Visibility data set not found - please verify the name'
# Write history to MS
try:
param_names = delmod.func_code.co_varnames[:delmod.func_code.
co_argcount]
param_vals = [eval(p) for p in param_names]
write_history(mstool(), vis, 'delmod', param_names, param_vals,
casalog)
except Exception, instance:
casalog.post("*** Error \'%s\' updating HISTORY" % (instance),
'WARN')
except Exception, instance:
print '*** Error ***', instance
def importfitsidi(fitsidifile,
vis,
constobsid=None,
scanreindexgap_s=None,
specframe=None):
"""Convert FITS-IDI visibility file into a CASA visibility file (MS).
Keyword arguments:
fitsidifile -- Name(s) of input FITS IDI file(s)
default: None; example='3C273XC1.IDI' or ['3C273XC1.IDI1', '3C273XC1.IDI2']
vis -- Name of output visibility file (MS)
default: None; example: vis='3C273XC1.ms'
constobsid -- If True a constant obs id == 0 of is given to all input files
default = False (new obs id for each input file)
scanreindexgap_s -- if > 0., a new scan is started whenever the gap between two
integrations is > the given value (seconds) or when a new field starts
or when the ARRAY_ID changes.
default = 0. (no reindexing)
specframe -- this frame will be used to set the spectral reference frame
for all spectral windows in the output MS
default = GEO (geocentric), other options: TOPO, LSRK, BARY
NOTE: if specframe is set to TOPO, the reference location will be taken from
the Observatories table in the CASA data repository for the given name of
the observatory. You can edit that table and add new rows.
"""
#Python script
retval = True
try:
casalog.origin('importfitsidi')
casalog.post("")
myms = mstool()
mytb = tbtool()
if type(specframe) == str and not specframe == '':
myspecframe = specframe.upper()
else:
myspecframe = 'GEO'
refframes = {
'REST': 0,
'LSRK': 1,
'LSRD': 2,
'BARY': 3,
'GEO': 4,
'TOPO': 5
}
if not refframes.has_key(myspecframe):
raise Exception, 'Value ' + myspecframe + ' of parameter specframe invalid. Possible values are REST, LSRK, LSRD, BARY, GEO, TOPO'
if (type(fitsidifile) == str):
casalog.post('### Reading file ' + fitsidifile, 'INFO')
myms.fromfitsidi(vis, fitsidifile)
myms.close()
elif (type(fitsidifile) == list):
clist = fitsidifile
casalog.post('### Reading file ' + clist[0], 'INFO')
myms.fromfitsidi(vis, clist[0])
myms.close()
clist.pop(0)
tname = vis + '_importfitsidi_tmp_'
shutil.rmtree(tname, ignore_errors=True)
for fidifile in clist:
casalog.post('### Reading file ' + fidifile, 'INFO')
myms.fromfitsidi(tname, fidifile)
myms.close()
myms.open(vis, nomodify=False)
myms.concatenate(msfile=tname, freqtol='', dirtol='')
myms.close()
shutil.rmtree(tname, ignore_errors=True)
else:
raise Exception, 'Parameter fitsidifile should be of type str or list'
if (constobsid):
mytb.open(vis + '/OBSERVATION', nomodify=False)
nobs = mytb.nrows()
cando = True
if nobs > 1:
casalog.post(
'Trying to keep obsid constant == 0 for all input files',
'INFO')
# check if all observations are from the same telescope; if not warn and leave as is
tels = mytb.getcol('TELESCOPE_NAME')
for i in range(1, nobs):
if tels[i] != tels[0]:
cando = False
if cando:
# get min and max time and write them into the first row;
casalog.post('Adjusting OBSERVATION table', 'INFO')
timeranges = mytb.getcol('TIME_RANGE')
ttr = timeranges.transpose()
newmin = min(ttr[0])
newmax = max(ttr[1])
mytb.putcell('TIME_RANGE', 0, [newmin, newmax])
# delete the other rows
mytb.removerows(range(1, nobs))
else:
casalog.post(
'The input files stem from different telescopes. Need to give different obs id.',
'WARN')
mytb.close()
if cando:
# give the same obs id == 0 to the entire output MS
casalog.post('Setting observation ID of all integrations to 0',
'INFO')
mytb.open(vis, nomodify=False)
for i in xrange(0, mytb.nrows()):
mytb.putcell('OBSERVATION_ID', i, 0)
mytb.close()
else: # don't want constant obs id
if (type(fitsidifile) == list and len(fitsidifile) > 1):
casalog.post(
'Incrementing observation ID for each input file ...',
'INFO')
if (scanreindexgap_s > 0.):
# reindex the scan column
mytb.open(vis, nomodify=False)
times = mytb.getcol('TIME')
fields = mytb.getcol('FIELD_ID')
arrayids = mytb.getcol('ARRAY_ID')
scannumbers = mytb.getcol('SCAN_NUMBER')
timesorted = np.argsort(np.array(times))
scannumber = 0
scannumber_field = len(fields) * [0]
prevtime = len(fields) * [0]
prevarrayid = arrayids[timesorted[0]]
for i in xrange(0, mytb.nrows()):
ii = timesorted[i]
timenow = times[ii]
fieldnow = fields[ii]
arrayidnow = arrayids[ii]
if (timenow-prevtime[fieldnow] > scanreindexgap_s) \
or (arrayidnow != prevarrayid):
scannumber += 1
scannumber_field[fieldnow] = scannumber
casalog.post("Starting new scan "+str(scannumber)+" at "+str(timenow)\
+", field "+str(fieldnow)+", array_id "+str(arrayidnow), 'INFO')
scannumbers[ii] = scannumber_field[fieldnow]
prevtime[fieldnow] = timenow
prevarrayid = arrayidnow
mytb.putcol('SCAN_NUMBER', scannumbers)
mytb.close()
if refframes.has_key(myspecframe):
casalog.post(
'Setting reference frame for all spectral windows to ' +
myspecframe, 'INFO')
if myspecframe == 'TOPO':
casalog.post(
'NOTE: reference position for TOPO frame will be the observatory location',
'WARN')
mytb.open(vis + '/SPECTRAL_WINDOW', nomodify=False)
refcol = mytb.getcol('MEAS_FREQ_REF')
refcol = [refframes[myspecframe]] * len(refcol)
mytb.putcol('MEAS_FREQ_REF', refcol)
mytb.close()
# write history
try:
param_names = importfitsidi.func_code.co_varnames[:importfitsidi.
func_code.
co_argcount]
param_vals = [eval(p) for p in param_names]
retval &= write_history(myms, vis, 'importfitsidi', param_names,
param_vals, casalog)
except Exception, instance:
casalog.post("*** Error \'%s\' updating HISTORY" % (instance),
'WARN')
except Exception, instance:
print '*** Error ***', instance
shutil.rmtree(vis + '_importfitsidi_tmp_', ignore_errors=True)
raise Exception, instance
def importatca(files=None,
vis=None,
options=None,
spw=None,
nscans=None,
lowfreq=None,
highfreq=None,
fields=None,
edge=8):
"""Convert an RPFITS file into a CASA visibility file (MS).
The conversion of the RPFITS format into a measurement set.
This version has been tested for both old ATCA and CABB data.
................
Keyword arguments:
files -- Name of input RPFITS file(s)
default: none; example: file='2010-01-02_1234.c999'
.... vis -- Output ms name, note a postfix (.ms) is NOT appended to this name
default: none
.... options -- Processing options, comma separated list
birdie - flag parts of spectrum known to be bad
reweight - (pre-CABB) reweight lag spectrum to avoid ringing
noautoflag - don't apply automatic flags (e.g. pointing scans)
noxycorr - don't apply xyphase correction
fastmosaic - use for large mosaics to speed up data access
hires - turn time binned data into fast sampled data
notsys - undo online Tsys calibration
noac - don't load autocorrelations
.... spw -- specify the input spectral windows to use. For CABB the order is
first continuum, 2nd continuum, then any zooms for first band,
followed by zooms for 2nd band. Pre-CABB data just has 0 and 1.
The output may have more spectral windows if there are frequency
changes.
........ default: all
.... nscans -- Number of scans to skip followed by number of scans to read
.... default: 0,0 (read all)
.... lowfreq -- Lowest reference frequency to select
.... default: 0 (all)
.... highfreq -- highest reference frequency to select
.... default: 0 (all)
.... fields -- List of field names to select
........ default: all
.... edge -- Percentage of edge channels to flag. For combined zooms, this
specifies the percentage for a single zoom
........ default: 8 (flags 4% of channels at lower and upper edge)
"""
# Python script
myaf = casac.atcafiller()
try:
try:
casalog.origin('importatca')
# -----------------------------------------
# beginning of importatca implementation
# -----------------------------------------
myaf.open(vis, files, options)
firstscan = 0
lastscan = 9999
if (nscans != None):
if len(nscans) > 0:
firstscan = nscans[0]
if len(nscans) > 1:
lastscan = nscans[1]
myaf.select(firstscan, lastscan, spw, lowfreq, highfreq, fields,
edge)
myaf.fill()
except Exception, e:
print e
casalog.post("Failed to import atca rpfits file(s) %s" % files)
raise
# Write the args to HISTORY.
try:
mslocal = mstool()
param_names = importatca.func_code.co_varnames[:importatca.
func_code.
co_argcount]
param_vals = [eval(p) for p in param_names]
write_history(mslocal, vis, 'importatca', param_names, param_vals,
casalog)
except Exception, instance:
casalog.post("Failed to updated HISTORY", 'WARN')
def uvcontsub3(vis, fitspw, combine, fitorder, field, spw,
scan, intent, correlation, observation):
"""Extract the line(s) of an MS."""
retval = True
casalog.origin('uvcontsub3')
myms = mstool()
mytb = tbtool()
# This one is redundant - it is already checked at the XML level.
if not ((type(vis) == str) and os.path.isdir(vis)):
casalog.post('Visibility data set not found - please verify the name', 'SEVERE')
return False
outputvis = vis + '.contsub'
if os.path.exists(outputvis):
casalog.post("Output MS " + outputvis + " already exists - will not overwrite.", 'SEVERE')
return False
if combine and combine.lower() != 'spw':
casalog.post("uvcontsub3 deliberately does not support combination by",
'SEVERE')
casalog.post("anything except spw.", 'SEVERE')
return False
# MSStateGram is picky ('CALIBRATE_WVR.REFERENCE, OBSERVE_TARGET_ON_SOURCE'
# doesn't work, but 'CALIBRATE_WVR.REFERENCE,OBSERVE_TARGET_ON_SOURCE'
# does), and I don't want to mess with bison now. A .upper() might be a
# good idea too, but the MS def'n v.2 does not say whether OBS_MODE should
# be case-insensitive.
intent = intent.replace(', ', ',')
if type(spw) == list:
spw = ','.join([str(s) for s in spw])
elif type(spw) == int:
spw = str(spw)
## if ':' in spw:
## casalog.post("uvcontsub3 does not yet support selection by channel for the output",
## 'SEVERE')
## casalog.post("Meanwhile, use split to select the desired channels", 'WARN')
## return False
if ';' in spw:
casalog.post("uvcontsub3 does not yet support writing multiple channel groups per output spw",
'SEVERE')
return False
mytb.open(vis + '/SPECTRAL_WINDOW')
allspw = '0~' + str(mytb.nrows() - 1)
mytb.close()
if 'spw' not in combine:
spwmfitspw = subtract_spws(spw, fitspw)
if spwmfitspw == 'UNKNOWN':
spwmfitspw = subtract_spws(allspw, fitspw)
if spwmfitspw:
raise Exception, "combine must include 'spw' when the fit is being applied to spws outside fitspw."
if type(correlation) == list:
correlation = ', '.join(correlation)
correlation = correlation.upper()
mytb.open(vis, nomodify=True)
if 'CORRECTED_DATA' in mytb.colnames():
datacolumn = 'CORRECTED_DATA'
else:
# DON'T remind the user that split before uvcontsub wastes time -
# scratch columns will eventually go away.
datacolumn = 'DATA'
mytb.close()
myms.open(vis, nomodify=True)
if not myms.contsub(outputms=outputvis, fitspw=fitspw,
fitorder=fitorder, combine=combine,
spw=spw, unionspw=join_spws(fitspw, spw),
field=field, scan=scan,
intent=intent, correlation=correlation,
obs=str(observation), whichcol=datacolumn):
myms.close()
return False
myms.close()
# Write history to output MS, not the input ms.
try:
param_names = uvcontsub3.func_code.co_varnames[:uvcontsub3.func_code.co_argcount]
param_vals = [eval(p) for p in param_names]
retval &= write_history(myms, outputvis, 'uvcontsub3', param_names, param_vals,
casalog)
except Exception, instance:
casalog.post("*** Error \'%s\' updating HISTORY" % (instance),
'WARN')
def applycal(
vis=None,
field=None,
spw=None,
intent=None,
selectdata=None,
timerange=None,
uvrange=None,
antenna=None,
scan=None,
observation=None,
msselect=None,
docallib=None,
callib=None,
gaintable=None,
gainfield=None,
interp=None,
spwmap=None,
calwt=None,
parang=None,
applymode=None,
flagbackup=None,
):
# Python script
casalog.origin('applycal')
# Take care of the trivial parallelization
if ParallelTaskHelper.isParallelMS(vis):
# Back up the flags, if requested (and if necessary)
if flagbackup and applymode != 'calonly' and applymode != 'trial':
fh.backupFlags(aflocal=None, msfile=vis, prename='applycal')
flagbackup = False
# To be safe convert file names to absolute paths.
gaintable = ParallelTaskHelper.findAbsPath(gaintable)
helper = ParallelTaskHelper('applycal', locals())
ret = helper.go()
if ParallelTaskHelper.getAsyncMode():
return ret
else:
return
try:
mycb = cbtool()
if (type(vis) == str) & os.path.exists(vis):
# add CORRECTED_DATA column
mycb.open(filename=vis, compress=False, addcorr=True,
addmodel=False)
else:
raise Exception, \
'Visibility data set not found - please verify the name'
# enforce default if unspecified
if applymode == '':
applymode = 'calflag'
# Back up the flags, if requested (and if necessary)
if flagbackup and applymode != 'calonly' and applymode \
!= 'trial':
fh.backupFlags(aflocal=None, msfile=vis, prename='applycal')
# Do data selection according to selectdata
if selectdata:
# pass all data selection parameters in as specified
mycb.selectvis(
time=timerange,
spw=spw,
scan=scan,
field=field,
intent=intent,
observation=str(observation),
baseline=antenna,
uvrange=uvrange,
chanmode='none',
msselect=msselect,
)
else:
# selectdata=F, so time,scan,baseline,uvrange,msselect=''
# using spw and field specifications only
mycb.selectvis(
time='',
spw=spw,
scan='',
field=field,
intent=intent,
observation='',
baseline='',
uvrange='',
chanmode='none',
msselect='',
)
# Arrange applies....
if docallib:
# by cal library from file
# parsing using c++ parser
thiscallib=mycb.parsecallibfile(callib)
mycb.setcallib(thiscallib)
else:
# by traditional parameters
ngaintab = 0
if gaintable != ['']:
ngaintab = len(gaintable)
ncalwt = len(calwt)
if ncalwt == 1:
calwt = [calwt[0] for i in range(ngaintab)]
ngainfld = len(gainfield)
nspwmap = len(spwmap)
ninterp = len(interp)
# handle list of list issues with spwmap
if nspwmap > 0:
if type(spwmap[0]) != list:
# first element not a list, only one spwmap specified
# make it a list of list
spwmap = [spwmap]
nspwmap = 1
for igt in range(ngaintab):
if gaintable[igt] != '':
# field selection is null unless specified
thisgainfield = ''
if igt < ngainfld:
thisgainfield = gainfield[igt]
# spwmap is null unless specifed
thisspwmap = [-1]
if igt < nspwmap:
thisspwmap = spwmap[igt]
# interp is 'linear' unless specified
thisinterp = 'linear'
if igt < ninterp:
if interp[igt] == '':
interp[igt] = thisinterp
thisinterp = interp[igt]
mycb.setapply(
t=0.0,
table=gaintable[igt],
field=thisgainfield,
calwt=calwt[igt],
spwmap=thisspwmap,
interp=thisinterp,
)
# ...and now the specialized terms
# Apply parallactic angle, if requested
if parang:
mycb.setapply(type='P')
mycb.correct(applymode)
# report what the flags did
reportflags(mycb.activityrec())
mycb.close()
# write history
try:
param_names = \
applycal.func_code.co_varnames[:applycal.func_code.co_argcount]
param_vals = [eval(p) for p in param_names]
write_history(
mstool(),
vis,
'applycal',
param_names,
param_vals,
casalog,
)
except Exception, instance:
casalog.post("*** Error \'%s\' updating HISTORY"
% instance, 'WARN')
except Exception, instance:
print '*** Error ***', instance
mycb.close()
casalog.post("Error in applycal: %s" % str(instance), "SEVERE")
raise Exception, "Error in applycal: "+str(instance)
def split_core(vis, outputvis, datacolumn, field, spw, width, antenna, timebin,
timerange, scan, intent, array, uvrange, correlation,
observation, combine, keepflags):
retval = True
if not outputvis or outputvis.isspace():
raise ValueError, 'Please specify outputvis'
myms = mstool()
mytb = None
if ((type(vis) == str) & (os.path.exists(vis))):
myms.open(vis, nomodify=True)
else:
raise ValueError, 'Visibility data set not found - please verify the name'
if os.path.exists(outputvis):
myms.close()
raise ValueError, "Output MS %s already exists - will not overwrite." % outputvis
if (os.path.exists(outputvis + ".flagversions")):
myms.close()
raise ValueError, "The flagversions \"%s.flagversions\" for the output MS already exist. Please delete." % outputvis
# No longer needed. When did it get put in? Note that the default
# spw='*' in myms.split ends up as '' since the default type for a variant
# is BOOLVEC. (Of course!) Therefore both split and myms.split must
# work properly when spw=''.
#if(spw == ''):
# spw = '*'
if (type(antenna) == list):
antenna = ', '.join([str(ant) for ant in antenna])
## Accept digits without units ...assume seconds
timebin = qa.convert(qa.quantity(timebin), 's')['value']
timebin = str(timebin) + 's'
if timebin == '0s':
timebin = '-1s'
# MSStateGram is picky ('CALIBRATE_WVR.REFERENCE, OBSERVE_TARGET_ON_SOURCE'
# doesn't work, but 'CALIBRATE_WVR.REFERENCE,OBSERVE_TARGET_ON_SOURCE'
# does), and I don't want to mess with bison now. A .upper() might be a
# good idea too, but the MS def'n v.2 does not say whether OBS_MODE should
# be case-insensitive.
intent = intent.replace(', ', ',')
if '^' in spw:
casalog.post(
"The interpretation of ^n in split's spw strings has changed from 'average n' to 'skip n' channels!",
'WARN')
casalog.post("Watch for Slicer errors", 'WARN')
if type(width) == str:
try:
if (width.isdigit()):
width = [string.atoi(width)]
elif (width.count('[') == 1 and width.count(']') == 1):
width = width.replace('[', '')
width = width.replace(']', '')
splitwidth = width.split(',')
width = []
for ws in splitwidth:
if (ws.isdigit()):
width.append(string.atoi(ws))
else:
width = [1]
except:
raise TypeError, 'parameter width is invalid...using 1'
if type(correlation) == list:
correlation = ', '.join(correlation)
correlation = correlation.upper()
if hasattr(combine, '__iter__'):
combine = ', '.join(combine)
if type(spw) == list:
spw = ','.join([str(s) for s in spw])
elif type(spw) == int:
spw = str(spw)
do_chan_mod = spw.find('^') > -1 # '0:2~11^1' would be pointless.
if not do_chan_mod: # ...look in width.
if type(width) == int and width > 1:
do_chan_mod = True
elif hasattr(width, '__iter__'):
for w in width:
if w > 1:
do_chan_mod = True
break
do_both_chan_and_time_mod = (do_chan_mod
and string.atof(timebin[:-1]) > 0.0)
if do_both_chan_and_time_mod:
# Do channel averaging first because it might be included in the spw
# string.
import tempfile
# We want the directory outputvis is in, not /tmp, because /tmp
# might not have enough space.
# outputvis is itself a directory, so strip off a trailing slash if
# it is present.
# I don't know if giving tempfile an absolute directory is necessary -
# dir='' is effectively '.' in Ubuntu.
workingdir = os.path.abspath(os.path.dirname(outputvis.rstrip('/')))
cavms = tempfile.mkdtemp(suffix=outputvis, dir=workingdir)
casalog.post('Channel averaging to ' + cavms)
if not myms.split(outputms=cavms,
field=field,
spw=spw,
step=width,
baseline=antenna,
subarray=array,
timebin='',
time=timerange,
whichcol=datacolumn,
scan=scan,
uvrange=uvrange,
combine=combine,
correlation=correlation,
intent=intent,
obs=str(observation)):
myms.close()
if os.path.isdir(cavms):
import shutil
shutil.rmtree(cavms)
return False
# The selection was already made, so blank them before time averaging.
field = ''
spw = ''
width = [1]
antenna = ''
array = ''
timerange = ''
datacolumn = 'all'
scan = ''
intent = ''
uvrange = ''
observation = ''
myms.close()
myms.open(cavms)
casalog.post('Starting time averaging')
if keepflags:
taqlstr = ''
else:
taqlstr = 'NOT (FLAG_ROW OR ALL(FLAG))'
if not myms.split(outputms=outputvis,
field=field,
spw=spw,
step=width,
baseline=antenna,
subarray=array,
timebin=timebin,
time=timerange,
whichcol=datacolumn,
scan=scan,
uvrange=uvrange,
combine=combine,
correlation=correlation,
taql=taqlstr,
intent=intent,
obs=str(observation)):
myms.close()
return False
myms.close()
if do_both_chan_and_time_mod:
import shutil
shutil.rmtree(cavms)
# Write history to output MS, not the input ms.
try:
param_names = split_core.func_code.co_varnames[:split_core.func_code.
co_argcount]
param_vals = [eval(p) for p in param_names]
retval &= write_history(myms, outputvis, 'oldsplit', param_names,
param_vals, casalog)
except Exception, instance:
casalog.post("*** Error \'%s\' updating HISTORY" % (instance), 'WARN')
casalog.post('Starting import ...', 'NORMAL')
importuvfits(fitsfile, vis)
except Exception, instance:
casalog.post(str(instance), 'SEVERE')
return retValue
# mytb, myms, myfg = gentools(['tb', 'ms', 'fg'])
mytb, myms = gentools(['tb', 'ms'])
aflocal = casac.agentflagger()
# Write history
try:
param_names = importgmrt.func_code.co_varnames[:importgmrt.func_code.
co_argcount]
param_vals = [eval(p) for p in param_names]
ok &= write_history(myms, vis, 'importgmrt', param_names, param_vals,
casalog)
except Exception, instance:
casalog.post("*** Error \'%s\' updating HISTORY" % (instance), 'WARN')
# CASA's importuvfits doesn't understand the GMRT antenna
# names very well and/or the FITS files tend to put the
# antenna names in a place that CASA doesn't look for them.
# Luckily the information seems to be in the "station"
# information. So let's fix up the Name column.
casalog.post('Correcting GMRT Antenna names.', 'NORMAL1')
try:
mytb.open(vis + '/ANTENNA', nomodify=False)
stations = mytb.getcol('STATION')
names = []
for idx in range(0, len(stations)):
names.append(stations[idx].split(':')[0])
# Open the MS, select the data and configure the output
mtlocal.open()
# Run the tool
casalog.post('Apply Hanning smoothing on data')
mtlocal.run()
mtlocal.done()
except Exception, instance:
mtlocal.done()
casalog.post('%s' % instance, 'ERROR')
return False
# Write history to output MS, not the input ms.
try:
param_names = hanningsmooth.func_code.co_varnames[:hanningsmooth.
func_code.
co_argcount]
param_vals = [eval(p) for p in param_names]
casalog.post('Updating the history in the output', 'DEBUG1')
write_history(mslocal, outputvis, 'hanningsmooth', param_names,
param_vals, casalog)
except Exception, instance:
casalog.post("*** Error \'%s\' updating HISTORY" % (instance), 'WARN')
return False
mslocal = None
return True
def importasdm(asdm=None,
vis=None,
createmms=None,
separationaxis=None,
numsubms=None,
corr_mode=None,
srt=None,
time_sampling=None,
ocorr_mode=None,
compression=None,
lazy=None,
asis=None,
wvr_corrected_data=None,
scans=None,
ignore_time=None,
process_syspower=None,
process_caldevice=None,
process_pointing=None,
process_flags=None,
tbuff=None,
applyflags=None,
savecmds=None,
outfile=None,
flagbackup=None,
verbose=None,
overwrite=None,
showversion=None,
useversion=None,
bdfflags=None,
with_pointing_correction=None,
remove_ref_undef=None,
convert_ephem2geo=None,
polyephem_tabtimestep=None):
"""Convert an ALMA Science Data Model observation into a CASA visibility file (MS) or single-dish data format (Scantable).
The conversion of the ALMA SDM archive format into a measurement set. This version
is under development and is geared to handling many spectral windows of different
shapes.
Keyword arguments:
asdm -- Name of input ASDM file (directory)
default: none; example: asdm='ExecBlock3'
vis -- Root ms or scantable name, note a prefix (.ms or .asap) is NOT appended to this name
default: none
createmms -- Create a Multi-MS
default: False
corr_mode -- correlation mode to be considered on input. Could
be one or more of the following, ao, co, ac, or all
default: all
srt -- spectral resolution type. Could be one or more of
the following, fr, ca, bw, or all
default: all
time_sampling -- specifies the time sampling, INTEGRATION and/or
SUBINTEGRATION. could be one or more of the following
i, si, or all.
default: all
ocorr_mode -- output data for correlation mode AUTO_ONLY
(ao) or CROSS_ONLY (co) or CROSS_AND_AUTO (ca)
default: ca
compression -- produces comrpressed columns in the resulting measurement set.
default: False
lazy -- Make the MS DATA column read the ASDM Binary data directly
(faster import, smaller MS)
default: False
asis -- creates verbatim copies of the ASDM tables in
the output measurement set. The value given to
this option must be a list of table names separated
by space characters; the wildcard character '*' is
allowed in table names.
wvr_corrected_data -- specifies which values are considered in the
ASDM binary data to fill the DATA column in
the MAIN table of the MS. Expected values for
this option are 'no' for the uncorrected data
(this is the default), 'yes' for the corrected
data and 'both' for corrected and uncorrected
data. In the latter case, two measurement sets
are created, one containing the uncorrected
data and the other one, whose name is suffixed
by '-wvr-corrected', containing the corrected
data.
scans -- processes only the scans specified in the option's value. This value is a semicolon
separated list of scan specifications. A scan specification consists in an exec bock index
followed by the character ':' followed by a comma separated list of scan indexes or scan
index ranges. A scan index is relative to the exec block it belongs to. Scan indexes are
1-based while exec blocks's are 0-based. "0:1" or "2:2~6" or "0:1,1:2~6,8;2:,3:24~30" "1,2"
are valid values for the option. "3:" alone will be interpreted as 'all the scans of the
exec block#3'. An scan index or a scan index range not preceded by an exec block index will
be interpreted as 'all the scans with such indexes in all the exec blocks'. By default
all the scans are considered.
ignore_time -- All the rows of the tables Feed, History, Pointing, Source, SysCal, CalDevice, SysPower,
and Weather are processed independently of the time range of the selected exec block / scan.
process_syspower -- The SysPower table is processed if and only if this parameter is set to True.
default: True
process_caldevice -- The CalDevice table is processed if and only if this parameter is set to True.
default: True
process_pointing -- The Pointing table is processed if and only if this parameter is set to True.
If the parameter is set to False the resulting MS will have an empty POINTING table.
default: True
process_flags -- Process the online flags and save them to the FLAG_CMD sub-table.
default: True
>>> process_flags expandable parameter
tbuff -- Time padding buffer (in seconds).
default: 0.0
applyflags -- Apply the online flags to the MS.
default: False
savecmds -- Save the online flags to an ASCII file.
default: False
outfile -- Filename to save the online flags.
default: ''
flagbackup -- Backup the FLAG column in the .flagversions.
default: True
verbose -- produce log output as asdm2MS is being run.
overwrite -- Over write an existing MS.
showversion -- report the version of the asdm2MS being used.
useversion -- Selects the version of asdm2MS to be used (presently only \'v3\' is available).
default: v3
bdfflags -- Set the MS FLAG column according to the ASDM _binary_ flags
default: false
with_pointing_correction -- add (ASDM::Pointing::encoder - ASDM::Pointing::pointingDirection)
to the value to be written in MS::Pointing::direction
default: false
remove_ref_undef -- if set to True then apply fixspwbackport on the resulting MSes.
convert_ephem2geo -- if True, convert any attached ephemerides to the GEO reference frame
polyephem_tabtimestep -- Timestep (days) for the tabulation of polynomial ephemerides. A value <= 0 disables tabulation.
Presently, VLA data can contain polynomial ephemerides. ALMA data uses tabulated values.
default: 0.
"""
# Python script
# make agentflagger tool local
aflocal = casac.agentflagger()
# make table tool local
tblocal = casac.table()
try:
casalog.origin('importasdm')
viso = ''
visoc = '' # for the wvr corrected version, if needed
if len(vis) > 0:
viso = vis
tmps = vis.rstrip('.ms')
if tmps == vis:
visoc = vis + '-wvr-corrected'
else:
visoc = tmps + '-wvr-corrected.ms'
else:
viso = asdm.rstrip("/") + '.ms'
visoc = asdm.rstrip("/") + '-wvr-corrected.ms'
vis = asdm.rstrip("/")
useversion = 'v3'
theexecutable = 'asdm2MS'
execute_string = theexecutable + ' --icm "' + corr_mode \
+ '" --isrt "' + srt + '" --its "' + time_sampling \
+ '" --ocm "' + ocorr_mode + '" --wvr-corrected-data "' \
+ wvr_corrected_data + '" --asis "' + asis \
+ '" --logfile "' + casalog.logfile() + '"'
if len(scans) > 0:
execute_string = execute_string + ' --scans ' + scans
if ignore_time:
execute_string = execute_string + ' --ignore-time'
if useversion == 'v3':
if not process_syspower:
execute_string = execute_string + ' --no-syspower'
if not process_caldevice:
execute_string = execute_string + ' --no-caldevice'
if not process_pointing:
execute_string = execute_string + ' --no-pointing'
if compression:
execute_string = execute_string + ' --compression'
elif lazy:
execute_string = execute_string + ' --lazy'
if verbose:
execute_string = execute_string + ' --verbose'
# if not overwrite and os.path.exists(viso):
# raise Exception, \
# 'You have specified an existing MS and have indicated you do not wish to overwrite it'
# Compression
if compression:
# viso = viso + '.compressed'
viso = viso.rstrip('.ms') + '.compressed.ms'
visoc = visoc.rstrip('.ms') + '.compressed.ms'
vistoproc = [] # the output MSs to post-process
if wvr_corrected_data == 'no' or wvr_corrected_data == 'both':
vistoproc.append(viso)
if (wvr_corrected_data == 'yes' or wvr_corrected_data == 'both'):
vistoproc.append(visoc)
for ff in vistoproc:
if not overwrite and os.path.exists(ff):
raise Exception, \
'You have specified an existing MS and have indicated you do not wish to overwrite it: %s'%ff
# If viso+".flagversions" then process differently depending on the value of overwrite..
#
if flagbackup:
for myviso in vistoproc:
dotFlagversion = myviso + '.flagversions'
if os.path.exists(dotFlagversion):
if overwrite:
casalog.post(
"Found '" + dotFlagversion +
"' . It'll be deleted before running the filler.")
os.system('rm -rf %s' % dotFlagversion)
else:
casalog.post("Found '%s' but can't overwrite it." %
dotFlagversion)
raise Exception, "Found '%s' but can't overwrite it." \
% dotFlagversion
# Make outfile always a list
if isinstance(outfile, str):
if outfile == '':
outfile = []
else:
noutfile = [outfile]
outfile = noutfile
if savecmds:
if len(outfile) == 0:
# Create default names for the online flags
for myviso in vistoproc:
outfile.append(myviso.replace('.ms', '_cmd.txt'))
elif len(outfile) != len(vistoproc):
casalog.post(
'List of outfile names does not match list of MSs', 'WARN')
casalog.post('Will save online flags to temporary filenames',
'WARN')
outfile = []
for myviso in vistoproc:
online_file = myviso.replace('.ms', '_TEMP_cmd.txt')
outfile.append(online_file)
if not overwrite:
for of in outfile:
if os.path.exists(of):
raise Exception, "Cannot overwrite online flags file '%s'; overwrite is set to False." % of
execute_string = execute_string + ' ' + asdm + ' ' + viso
if showversion:
casalog.post(
"You set option \'showversion\' to True. Will just output the version information and then terminate.",
'WARN')
execute_string = theexecutable + ' --revision'
if with_pointing_correction:
execute_string = execute_string + ' --with-pointing-correction'
if (polyephem_tabtimestep !=
None) and (type(polyephem_tabtimestep) == int
or type(polyephem_tabtimestep) == float):
if polyephem_tabtimestep > 0:
casalog.post(
'Will tabulate all attached polynomial ephemerides with a time step of '
+ str(polyephem_tabtimestep) + ' days.')
if polyephem_tabtimestep > 1.:
casalog.post(
'A tabulation timestep of <= 1 days is recommended.',
'WARN')
execute_string = execute_string + ' --polyephem-tabtimestep ' + str(
polyephem_tabtimestep)
casalog.post('Running ' + theexecutable + ' standalone invoked as:')
# print execute_string
casalog.post(execute_string)
exitcode = os.system(execute_string)
if exitcode != 0:
if not showversion:
casalog.post(
theexecutable + ' terminated with exit code ' +
str(exitcode), 'SEVERE')
raise Exception, \
'ASDM conversion error. Please check if it is a valid ASDM and that data/alma/asdm is up to date.'
if showversion:
return
#
# Possibly remove the name of the measurement set expected to contain the corrected data from the list of of produced measurement
# sets if it appears the filler did not find any corrected data.
#
if not os.path.exists(visoc):
vistoproc = [myviso for myviso in vistoproc if myviso != visoc]
#
# Do we apply fixspwbackport
if remove_ref_undef:
casalog.post(
'remove_ref_undef=True: fixspwbackport will be applied ...')
for myviso in vistoproc:
cmd = 'fixspwbackport ' + myviso
casalog.post('Running fixspwbackport standalone invoked as:')
casalog.post(cmd)
cmdexitcode = os.system(cmd)
if cmdexitcode != 0:
casalog.post(
cmd + ' terminated with exit code ' + str(cmdexitcode),
'SEVERE')
raise Exception, 'fixspwbackport error.'
# Binary Flag processing
if bdfflags:
casalog.post(
'Parameter bdfflags==True: flags from the ASDM binary data will be used to set the MS flags ...'
)
bdffexecutable = 'bdflags2MS '
bdffexecstring_base = bdffexecutable + ' -f ALL' + ' --ocm "' + ocorr_mode \
+ '" --logfile "' + casalog.logfile() + '"'
if len(scans) > 0:
bdffexecstring_base = bdffexecstring_base + ' --scans ' + scans
if lazy and not compression:
bdffexecstring_base = bdffexecstring_base + ' --lazy=true'
for myviso in vistoproc:
if myviso.find("wvr-corrected") != -1:
options = " --wvr-corrected=True "
else:
options = " "
bdffexecstring = bdffexecstring_base + options + asdm + ' ' + myviso
casalog.post('Running ' + bdffexecutable +
' standalone invoked as:')
casalog.post(bdffexecstring)
bdffexitcode = os.system(bdffexecstring)
if bdffexitcode != 0:
casalog.post(
bdffexecutable + ' terminated with exit code ' +
str(bdffexitcode), 'SEVERE')
raise Exception, \
'ASDM binary flags conversion error. Please check if it is a valid ASDM and that data/alma/asdm is up to date.'
theephemfields = ce.findattachedephemfields(myviso, field='*')
if len(theephemfields) > 0:
# until asdm2MS does this internally: recalc the UVW coordinates for ephem fields
imt = imtool()
imt.open(myviso, usescratch=False)
imt.calcuvw(theephemfields, refcode='J2000', reuse=False)
imt.close()
if convert_ephem2geo:
for myviso in vistoproc:
ce.convert2geo(myviso,
'*') # convert any attached ephemerides to GEO
if len(theephemfields) > 0:
# also set the direction column in the SOURCE table
tblocal.open(myviso + '/FIELD', nomodify=False)
sourceids = tblocal.getcol('SOURCE_ID')
ftimes = tblocal.getcol('TIME')
ftimekw = tblocal.getcolkeywords('TIME')
tmpa = tblocal.getcol('PHASE_DIR')
origphasedir = tmpa
affectedsids = []
thesamplefields = []
for fld in theephemfields: # determine all source ids used by the ephem fields
if not (sourceids[fld]
in affectedsids): # this source id wasn't handled yet
affectedsids.append(sourceids[fld])
thesamplefields.append(fld)
# need to temporarily change the offset (not all mosaics have an element at (0,0))
tmpa[0][0][fld] = 0.
tmpa[1][0][fld] = 0.
#endif
#endfor
tblocal.putcol('PHASE_DIR', tmpa)
tblocal.close()
tblocal.open(myviso + '/SOURCE')
sourceposref = tblocal.getcolkeywords(
'DIRECTION')['MEASINFO']['Ref']
tblocal.close()
directions = []
msmdlocal = casac.msmetadata()
msmdlocal.open(myviso)
for fld in thesamplefields:
thedirmeas = msmdlocal.phasecenter(fld)
if thedirmeas['refer'] != sourceposref:
casalog.post(
'Ephemeris is in ' + thedirmeas['refer'] +
' instead of ' + sourceposref +
' frame.\nEntry in SOURCE table will be converted to '
+ sourceposref, 'WARN')
melocal = metool()
melocal.doframe(thedirmeas)
thedirmeas = melocal.measure(thedirmeas, sourceposref)
directions.append(
[thedirmeas['m0']['value'], thedirmeas['m1']['value']])
thetime = me.epoch(v0=str(ftimes[fld]) + 's',
rf=ftimekw['MEASINFO']['Ref'])
casalog.post("Will set SOURCE direction for SOURCE_ID " +
str(sourceids[fld]) +
" to ephemeris phase center for time " +
str(thetime['m0']['value']) + " " +
thetime['m0']['unit'] + " " + thetime['refer'])
#endfor
msmdlocal.close()
# restore original PHASE_DIR
tblocal.open(myviso + '/FIELD', nomodify=False)
tblocal.putcol('PHASE_DIR', origphasedir)
tblocal.close()
# write source directions
tblocal.open(myviso + '/SOURCE', nomodify=False)
ssourceids = tblocal.getcol('SOURCE_ID')
sdirs = tblocal.getcol('DIRECTION')
for row in xrange(0, len(ssourceids)):
for i in xrange(0, len(affectedsids)):
if ssourceids[row] == affectedsids[i]:
sdirs[0][row] = directions[i][0]
sdirs[1][row] = directions[i][1]
break
#endfor
#endfor
tblocal.putcol('DIRECTION',
sdirs) # write back corrected directions
tblocal.close()
#end if
##############################################################################################3
# CAS-7369 - Create an output Multi-MS (MMS)
if createmms:
# Get the default parameters of partition
from tasks import partition
fpars = partition.parameters
for mypar in fpars.keys():
fpars[mypar] = partition.itsdefault(mypar)
# Call the cluster for each MS
for myviso in vistoproc:
casalog.origin('importasdm')
# Move original MS to tempdir
tempname = myviso + '.temp.ms'
outputmms = myviso
shutil.move(myviso, tempname)
# Get the proper column
datacolumn = 'DATA'
dcols = ['DATA', 'FLOAT_DATA']
for dc in dcols:
if len(th.getColDesc(tempname, dc)) > 0:
datacolumn = dc
break
fpars['datacolumn'] = datacolumn
casalog.post('Will create a Multi-MS for: ' + myviso)
fpars['vis'] = tempname
fpars['flagbackup'] = False
fpars['outputvis'] = outputmms
fpars['separationaxis'] = separationaxis
fpars['numsubms'] = numsubms
# Run partition only at the MPIServers
pdh = ParallelDataHelper('partition', fpars)
# Get a cluster
pdh.setupCluster(thistask='partition')
try:
pdh.go()
# Remove original MS
shutil.rmtree(tempname)
except Exception, instance:
# Restore MS in case of error in MMS creation
shutil.move(tempname, myviso)
casalog.post('%s' % instance, 'ERROR')
return False
casalog.origin('importasdm')
# Create a .flagversions for the MS or MMS
if flagbackup:
for myviso in vistoproc:
if os.path.exists(myviso):
aflocal.open(myviso)
aflocal.saveflagversion(
'Original',
comment='Original flags at import into CASA',
merge='save')
aflocal.done()
# Importasdm Flag Parsing
if os.access(asdm + '/Flag.xml', os.F_OK):
# Find Flag.xml
casalog.post('Found Flag.xml in SDM')
# Find Antenna.xml
if os.access(asdm + '/Antenna.xml', os.F_OK):
casalog.post('Found Antenna.xml in SDM')
else:
raise Exception, 'Failed to find Antenna.xml in SDM'
# Find SpectralWindow.xml
if os.access(asdm + '/SpectralWindow.xml', os.F_OK):
casalog.post('Found SpectralWindow.xml in SDM')
else:
raise Exception, \
'Failed to find SpectralWindow.xml in SDM'
#
# Parse Flag.xml into flag dictionary
#
if process_flags:
flagcmds = fh.parseXML(asdm, float(tbuff))
onlinekeys = flagcmds.keys()
nflags = onlinekeys.__len__()
# Apply flags to the MS
if nflags > 0:
idx = 0
for myviso in vistoproc:
if applyflags:
# Open the MS and attach it to the tool
aflocal.open(myviso)
# Select the data
aflocal.selectdata()
# Setup the agent's parameters
fh.parseAgents(aflocal, flagcmds, [], True, True,
'')
# Initialize the agents
aflocal.init()
# Run the tool
aflocal.run(True, True)
casalog.post('Applied %s flag commands to %s' %
(nflags, myviso))
# Destroy the tool and de-attach the MS
aflocal.done()
# Save to FLAG_CMD table. APPLIED is set to True.
fh.writeFlagCommands(myviso, flagcmds, True, '',
'', True)
else:
casalog.post(
'Will not apply flags to %s (apply_flags=False), use flagcmd to apply'
% myviso)
# Write to FLAG_CMD, APPLIED is set to False
fh.writeFlagCommands(myviso, flagcmds, False, '',
'', True)
# Save the flag cmds to an ASCII file
if savecmds:
# Save to standard filename
fh.writeFlagCommands(myviso, flagcmds, False, '',
outfile[idx], False)
casalog.post('Saved %s flag commands to %s' %
(nflags, outfile[idx]))
idx += 1
else:
casalog.post('There are no flag commands to process')
else:
casalog.post('There is no Flag.xml in ASDM', 'WARN')
# Write parameters to HISTORY table of MS
mslocal = mstool()
param_names = importasdm.func_code.co_varnames[:importasdm.func_code.
co_argcount]
param_vals = [eval(p) for p in param_names]
for myviso in vistoproc:
write_history(mslocal, myviso, 'importasdm', param_names,
param_vals, casalog)
return
def fluxscale(vis=None,
caltable=None,
fluxtable=None,
reference=None,
transfer=None,
listfile=None,
append=None,
refspwmap=None,
gainthreshold=None,
antenna=None,
timerange=None,
scan=None,
incremental=None,
fitorder=None,
display=None):
"""Bootstrap the flux density scale from standard calibrators:
After running gaincal on standard flux density calibrators (with or
without a model), and other calibrators with unknown flux densities,
fluxscale will determine the flux density of the unknowns calibrators
that are most consistent with the standard calibrator antenna gains.
Keyword arguments:
vis -- Name of input visibility file
default: none; example: vis='ngc5921.ms'
caltable -- Name of input calibration table
default: none; example: caltable='ngc5921.gcal'
This cal table was obtained from task gaincal.
fluxtable -- Name of output, flux-scaled calibration table
default: none; example: fluxtable='ngc5921.gcal2'
The gains in this table have been adjusted by the
derived flux density each calibrator. The MODEL_DATA
column has NOT been updated for the flux density of the
calibrator. Use setjy to do this if it is a point source.
reference -- Reference field name(s)
The names of the fields with a known flux densities or
visibilties that have been placed in the MODEL column
by setjy or ft for a model not in the CASA system.
The syntax is similar to field. Hence field index or
names can be used.
default: none; example: reference='1328+307'
transfer -- Transfer field name(s)
The names of the fields with unknown flux densities.
These should be point-like calibrator sources
The syntax is similar to field. Hence source index or
names can be used.
default: '' = all sources in caltable that are not specified
as reference sources. Do not include unknown target sources
example: transfer='1445+099, 3C84'; transfer = '0,4'
NOTE: All fields in reference and transfer must have solutions
in the caltable.
listfile -- Fit listfile name
The list file contains the flux density, flux density error,
S/N, and number of solutions (all antennas and feeds) for each
spectral window. NOTE: The nominal spectral window frequencies
will be included in the future.
default: '' = no fit listfile will be created.
append -- Append fluxscaled solutions to the fluxtable.
default: False; (will overwrite if already existing)
example: append=True
refspwmap -- Vector of spectral windows enablings scaling across
spectral windows
default: [-1]==> none.
Example with 4 spectral windows:
if the reference fields were observed only in spw=1 & 3,
and the transfer fields were observed in all 4 spws (0,1,2,3),
specify refspwmap=[1,1,3,3].
This will ensure that transfer fields observed in spws 0,1,2,3
will be referenced to reference field solutions only in
spw 1 or 3.
gainthreshold -- % deviation threshold from the median gain to be used flux scaling derivation
antenna -- Select antennas to be used to derive flux scaling
timerange -- Select timerage to be used to derive flux scaling with given antenna selection
scan -- Select scans to be used to derived flux scaling with given antenna selection
incremental -- Create an incremental caltable containing only the gain correction
factors.
default: False; (older behavior, produces flux scaled gain table)
fitorder -- the order of spectral fitting when solutions for multiple spws are available
default: 1
display -- display statistics (histogram) of derived correction factors
default: false
"""
try:
casalog.origin('fluxscale')
mycb = cbtool()
mycb.open(filename=vis, compress=False, addcorr=False, addmodel=False)
output = mycb.fluxscale(tablein=caltable,
tableout=fluxtable,
reference=reference,
transfer=transfer,
listfile=listfile,
append=append,
refspwmap=refspwmap,
gainthreshold=gainthreshold,
antenna=antenna,
timerange=timerange,
scan=scan,
incremental=incremental,
fitorder=fitorder,
display=display)
mycb.close()
#write history
try:
param_names = fluxscale.func_code.co_varnames[:fluxscale.func_code.
co_argcount]
param_vals = [eval(p) for p in param_names]
write_history(mstool(), vis, 'fluxscale', param_names, param_vals,
casalog)
writeResultsHistory(mstool(), vis, casalog, output)
except Exception, instance:
casalog.post("*** Error \'%s\' updating HISTORY" % (instance),
'WARN')
except Exception, instance:
print '*** Error ***', instance
mycb.close()
raise Exception, instance
def concat(vislist,concatvis,freqtol,dirtol,respectname,timesort,copypointing,
visweightscale, forcesingleephemfield):
"""concatenate visibility datasets
The list of data sets given in the vis argument are chronologically concatenated
into an output data set in concatvis, i.e. the data sets in vis are first ordered
by the time of their earliest integration and then concatenated.
If there are fields whose direction agrees within the direction tolerance
(parameter dirtol), the actual direction in the resulting, merged output field
will be the one from the chronologically first input MS.
If concatvis already exists (e.g., it is the same as the first input data set),
then the other input data sets will be appended to the concatvis data set.
There is no limit to the number of input data sets.
If none of the input data sets have any scratch columns (model and corrected
columns), none are created in the concatvis. Otherwise these columns are
created on output and initialized to their default value (1 in model column,
data in corrected column) for those data with no input columns.
Spectral windows for each data set with the same chanelization, and within a
specified frequency tolerance of another data set will be combined into one
spectral window.
A field position in one data set that is within a specified direction tolerance
of another field position in any other data set will be combined into one
field. The field names need not be the same---only their position is used.
Each appended dataset is assigned a new observation id (provided the entries
in the observation table are indeed different).
Keyword arguments:
vis -- Name of input visibility files to be combined
default: none; example: vis = ['src2.ms','ngc5921.ms','ngc315.ms']
concatvis -- Name of visibility file that will contain the concatenated data
note: if this file exits on disk then the input files are
added to this file. Otherwise the new file contains
the concatenated data. Be careful here when concatenating to
an existing file.
default: none; example: concatvis='src2.ms'
example: concatvis='outvis.ms'
other examples:
concat(vis=['src2.ms','ngc5921.ms'], concatvis='src2.ms')
will concatenate 'ngc5921.ms' into 'src2.ms', and the original
src2.ms is lost
concat(vis=['src2.ms','ngc5921.ms'], concatvis='out.ms')
will concatenate 'ngc5921.ms' and 'src2.ms' into a file named
'out.ms'; the original 'ngc5921.ms' and 'src2.ms' are untouched.
concat(vis=['v1.ms','v2.ms'], concatvis = 'vall.ms')
then
concat(vis=['v3.ms','v4.ms'], concatvis = 'vall.ms')
vall.ms will contains v1.ms+v2.ms+v3.ms+v4.ms
Note: run flagmanager to save flags in the concatvis
freqtol -- Frequency shift tolerance for considering data to be in the same
spwid. The number of channels must also be the same.
default: '' == 1 Hz
example: freqtol='10MHz' will not combine spwid unless they are
within 10 MHz.
Note: This option is useful to combine spectral windows with very slight
frequency differences caused by Doppler tracking, for example.
dirtol -- Direction shift tolerance for considering data as the same field
default: '' == 1 mas (milliarcsec)
example: dirtol='1.arcsec' will not combine data for a field unless
their phase center differ by less than 1 arcsec. If the field names
are different in the input data sets, the name in the output data
set will be the first relevant data set in the list.
respectname -- If true, fields with a different name are not merged even if their
direction agrees (within dirtol)
default: False
timesort -- If true, the output visibility table will be sorted in time.
default: false. Data in order as read in.
example: timesort=true
Note: There is no constraint on data that is simultaneously observed for
more than one field; for example multi-source correlation of VLBA data.
copypointing -- Make a proper copy of the POINTING subtable (can be time consuming).
If False, the result is an empty POINTING table.
default: True
visweightscale -- The weights of the individual MSs will be scaled in the concatenated
output MS by the factors in this list. Useful for handling heterogeneous arrays.
Use plotms to inspect the "Wt" column as a reference for determining the scaling
factors. See the cookbook for more details.
example: [1.,3.,3.] - scale the weights of the second and third MS by a factor 3.
default: [] (empty list) - no scaling
forcesingleephemfield -- By default, concat will only merge two ephemeris fields if
the first ephemeris covers the time range of the second. Otherwise, two separate
fields with separate ephemerides are placed in the output MS.
In order to override this behaviour and make concat merge the non-overlapping
or only partially overlapping input ephemerides, the name or id of the field
in question needs to be placed into the list in parameter 'forcesingleephemfield'.
example: ['Neptune'] - will make sure that there is only one joint ephemeris for
field Neptune in the output MS
default: '' - standard treatment of all ephemeris fields
"""
###
#Python script
try:
casalog.origin('concat')
t = tbtool()
m = mstool()
#break the reference between vis and vislist as we modify vis
if(type(vislist)==str):
vis=[vislist]
else:
vis=list(vislist)
#dto. for concavis
theconcatvis = concatvis
# warn if there are MMSs
mmslist = []
for elvis in vis : ###Oh no Elvis does not exist Mr Bill
if(ParallelTaskHelper.isParallelMS(elvis)):
mmslist.append(elvis)
if len(mmslist)>0:
if (vis[0] == mmslist[0]):
casalog.post('*** The first input MS is a multi-MS to which no row can be added. Cannot proceed.', 'WARN')
casalog.post('*** Please use virtualconcat or convert the first input MS to a normal MS using split.', 'WARN')
raise Exception, 'Cannot append to a multi-MS. Please use virtualconcat.'
casalog.post('*** The following input measurement sets are multi-MSs', 'INFO')
for mname in mmslist:
casalog.post('*** '+mname, 'INFO')
casalog.post('*** Use virtualconcat to produce a single multi-MS from several multi-MSs.', 'INFO')
doweightscale = False
if(len(visweightscale)>0):
if (len(visweightscale) != len(vis)):
raise Exception, 'parameter visweightscale must have same number of elements as parameter vis'
for factor in visweightscale:
if factor<0.:
raise Exception, 'parameter visweightscale must only contain positive numbers'
elif factor!=1.:
doweightscale=True
# process the input MSs in chronological order
sortedvis = []
sortedvisweightscale = []
namestuples = []
for name in vis:
t.open(name)
times = t.getcol('TIME')
t.close()
times.sort()
if doweightscale:
namestuples.append( (times[0], name, visweightscale[vis.index(name)]) )
else:
namestuples.append( (times[0], name, 0) )
sorted_namestuples = sorted(namestuples, key=lambda msname: msname[0])
for i in range(0,len(vis)):
sortedvis.append(sorted_namestuples[i][1])
sortedvisweightscale.append(sorted_namestuples[i][2])
if((type(concatvis)!=str) or (len(concatvis.split()) < 1)):
raise Exception, 'parameter concatvis is invalid'
existingconcatvis = False
if(vis.count(concatvis) > 0):
existingconcatvis = True
cvisindex = sortedvis.index(concatvis)
if not sorted_namestuples[cvisindex][0] == sorted_namestuples[0][0]:
raise Exception, 'If concatvis is set to the name of an existing MS in vis, it must be the chronologically first.'+\
'\n I.e. in this case you should set concatvis to '+sortedvis[0]
sortedvis.pop(cvisindex)
if doweightscale:
vwscale = sortedvisweightscale[cvisindex]
sortedvisweightscale.pop(cvisindex)
sortedvisweightscale = [vwscale] + sortedvisweightscale # move the corresponding weight to the front
if not vis == sortedvis:
casalog.post('The list of input MSs is not in chronological order and will need to be sorted.' , 'INFO')
casalog.post('The chronological order in which the concatenation will take place is:' , 'INFO')
if existingconcatvis:
casalog.post(' MJD '+str(qa.splitdate(qa.quantity(sorted_namestuples[0][0],'s'))['mjd'])+': '+concatvis, 'INFO')
for name in sortedvis:
casalog.post(' MJD '+str(qa.splitdate(qa.quantity(sorted_namestuples[sortedvis.index(name)][0],'s'))['mjd'])+': '+name, 'INFO')
if doweightscale:
casalog.post('In this new order, the weights are:'+str(sortedvisweightscale) , 'INFO')
# replace the original vis and visweightscale by the sorted ones (with concatvis removed if it exists)
vis = sortedvis
visweightscale = sortedvisweightscale
if(os.path.exists(concatvis)):
casalog.post('Will be concatenating into the existing ms '+concatvis , 'WARN')
if doweightscale and not existingconcatvis:
visweightscale = [1.]+visweightscale # set the weight for this existing MS to 1.
casalog.post('The weights for this existing MS will be left unchanged.' , 'WARN')
else:
if(len(vis) >0): # (note: in case len is 1, we only copy, essentially)
casalog.post('copying '+vis[0]+' to '+theconcatvis , 'INFO')
shutil.copytree(vis[0], theconcatvis)
# note that the resulting copy is writable even if the original was read-only
vis.pop(0)
# don't need to pop visweightscale here!
if not copypointing: # remove the rows from the POINTING table of the first MS
casalog.post('*** copypointing==False: resulting MS will have empty POINTING table.', 'INFO')
tmptabname = 'TMPPOINTING'+str(time.time())
shutil.rmtree(tmptabname, ignore_errors=True)
shutil.move(theconcatvis+'/POINTING', tmptabname)
t.open(tmptabname)
if(t.nrows()>0):
ttab = t.copy(newtablename=theconcatvis+'/POINTING', deep=False, valuecopy=True, norows=True)
ttab.close()
t.close()
shutil.rmtree(tmptabname, ignore_errors=True)
else: # the POINTING table is already empty
casalog.post('*** Input POINTING table was already empty.', 'INFO')
shutil.move(tmptabname, theconcatvis+'/POINTING')
t.close()
# handle the ephemeris concatenation
if not forcesingleephemfield=='':
from recipes.ephemerides import concatephem
if type(forcesingleephemfield)==str or type(forcesingleephemfield)==int:
forcesingleephemfield = [forcesingleephemfield]
if not type(forcesingleephemfield) == list:
raise Exception, 'Type of parameter forcesingleephemfield must be str, int, or list'
themss = [theconcatvis]
for x in vis:
themss.append(x)
for ephemfield in forcesingleephemfield:
if not type(ephemfield)==int:
ephemfield = str(ephemfield)
casalog.post('*** Forcing single ephemeris for field '+str(ephemfield), 'INFO')
thetabs = concatephem.findephems(themss, ephemfield)
if thetabs != [] and not ('' in thetabs):
tmptab = os.path.basename(thetabs[0])+'.concattmp'
targettab = theconcatvis+'/FIELD/'+os.path.basename(thetabs[0])
if not os.path.exists(targettab):
raise Exception, 'Internal ERROR: ephemeris '+targettab+' does not exist'
concatephem.concatephem(thetabs, tmptab)
if os.path.exists(tmptab):
os.system('rm -rf '+targettab)
os.system('mv '+tmptab+' '+targettab)
else:
casalog.post('ERROR while forcing single ephemeris for field '+str(ephemfield), 'SEVERE')
raise Exception, 'Concatenation of ephemerides for field '+str(ephemfield)+' failed.'
else:
casalog.post('ERROR while forcing single ephemeris for field '+str(ephemfield), 'SEVERE')
raise Exception, 'Cannot find ephemerides for field '+str(ephemfield)+' in all input MSs.'
# Determine if scratch columns should be considered at all
# by checking if any of the MSs has them.
considerscrcols = False
considercorr = False
considermodel = False
needscrcols = []
needmodel = []
needcorr = []
if ((type(theconcatvis)==str) and (os.path.exists(theconcatvis))):
# check if all scratch columns are present
t.open(theconcatvis)
if (t.colnames().count('MODEL_DATA')==1):
considermodel = True
if(t.colnames().count('CORRECTED_DATA')==1):
considercorr = True
needscrcols.append(t.colnames().count('CORRECTED_DATA')==0
or t.colnames().count('MODEL_DATA')==0)
needmodel.append(t.colnames().count('MODEL_DATA')==0)
needcorr.append(t.colnames().count('CORRECTED_DATA')==0)
t.close()
else:
raise Exception, 'Visibility data set '+theconcatvis+' not found - please verify the name'
for elvis in vis : ###Oh no Elvis does not exist Mr Bill
if(not os.path.exists(elvis)):
raise Exception, 'Visibility data set '+elvis+' not found - please verify the name'
# check if all scratch columns are present
t.open(elvis)
if (t.colnames().count('MODEL_DATA')==1):
considermodel = True
if(t.colnames().count('CORRECTED_DATA')==1):
considercorr = True
needscrcols.append(t.colnames().count('CORRECTED_DATA')==0
or t.colnames().count('MODEL_DATA')==0)
needmodel.append(t.colnames().count('MODEL_DATA')==0)
needcorr.append(t.colnames().count('CORRECTED_DATA')==0)
t.close()
considerscrcols = (considercorr or considermodel) # there are scratch columns
# start actual work, file existence has already been checked
i = 0
if(considerscrcols and needscrcols[i]):
# create scratch cols
casalog.post('creating scratch columns in '+theconcatvis , 'INFO')
_cb.open(theconcatvis,
addcorr=(considercorr and needcorr[i]),
addmodel=(considermodel and needmodel[i])) # calibrator-open creates scratch columns
_cb.close()
# scale the weights and sigma of the first MS in the chain
if doweightscale:
wscale = visweightscale[i]
if(wscale==1.):
casalog.post('Will leave the weights for this MS unchanged.', 'INFO')
else:
casalog.post('Scaling weights for first MS by factor '+str(wscale), 'INFO')
t.open(theconcatvis, nomodify=False)
for colname in [ 'WEIGHT', 'WEIGHT_SPECTRUM']:
if (colname in t.colnames()) and (t.iscelldefined(colname,0)):
for j in xrange(0,t.nrows()):
a = t.getcell(colname, j)
a *= wscale
t.putcell(colname, j, a)
for colname in ['SIGMA']:
if (wscale > 0. and colname in t.colnames()) and (t.iscelldefined(colname,0)):
sscale = 1./sqrt(wscale)
for j in xrange(0,t.nrows()):
a = t.getcell(colname, j)
a *= sscale
t.putcell(colname, j, a)
t.close()
# determine handling switch value
handlingswitch = 0
if not copypointing:
handlingswitch = 2
m.open(theconcatvis,nomodify=False)
mmsmembers = [theconcatvis]
for elvis in vis :
i = i + 1
destms = ""
casalog.post('concatenating '+elvis+' into '+theconcatvis , 'INFO')
wscale = 1.
if doweightscale:
wscale = visweightscale[i]
if(wscale==1.):
casalog.post('Will leave the weights for this MS unchanged.', 'INFO')
else:
casalog.post('Will scale weights for this MS by factor '+str(wscale) , 'INFO')
if(considerscrcols and needscrcols[i]):
if(ParallelTaskHelper.isParallelMS(elvis)):
raise Exception, 'Cannot create scratch columns in a multi-MS. Use virtualconcat.'
else:
# create scratch cols
casalog.post('creating scratch columns for '+elvis+' (original MS unchanged)', 'INFO')
tempname = elvis+'_with_scrcols'
shutil.rmtree(tempname, ignore_errors=True)
shutil.copytree(elvis, tempname)
_cb.open(tempname,
addcorr=(considercorr and needcorr[i]),
addmodel=(considermodel and needmodel[i])) # calibrator-open creates scratch columns
_cb.close()
# concatenate copy instead of original file
m.concatenate(msfile=tempname,freqtol=freqtol,dirtol=dirtol,respectname=respectname,
weightscale=wscale,handling=handlingswitch,
destmsfile=destms)
shutil.rmtree(tempname, ignore_errors=True)
else:
m.concatenate(msfile=elvis,freqtol=freqtol,dirtol=dirtol,respectname=respectname,
weightscale=wscale,handling=handlingswitch,
destmsfile=destms)
if timesort:
casalog.post('Sorting main table by TIME ...', 'INFO')
m.timesort()
# Write history to output MS, not the input ms.
try:
param_names = concat.func_code.co_varnames[:concat.func_code.co_argcount]
param_vals = [eval(p) for p in param_names]
write_history(mstool(), concatvis, 'concat', param_names,
param_vals, casalog)
except Exception, instance:
casalog.post("*** Error \'%s\' updating HISTORY" % (instance),
'WARN')
m.close()
return True
def cvel(vis, outputvis, passall, field, spw, selectdata, antenna, timerange,
scan, array, mode, nchan, start, width, interpolation, phasecenter,
restfreq, outframe, veltype, hanning):
""" regrid an MS to a new spectral window / channel structure or frame
vis -- Name of input visibility file
default: none; example: vis='ngc5921.ms'
outputvis -- Name of output measurement set (required)
default: none; example: vis='ngc5921-regridded.ms'
passall -- if False, data not meeting the selection on field or spw
is omitted; if True, data not meeting the selection
is passed through without modification
default: False; example:
field='NGC5921'
passall=False : only data from NGC5921 is included in output MS,
no data from other fields (e.g. 1331+305) is included
passall=True : data from NGC5921 is transformed by cvel, all other
fields are passed through unchanged
field -- Select fields in MS. Use field id(s) or field name(s).
['go listobs' to obtain the list id's or names]
default: ''= all fields
If field string is a non-negative integer, it is assumed to
be a field index otherwise, it is assumed to be a
field name
field='0~2'; field ids 0,1,2
field='0,4,5~7'; field ids 0,4,5,6,7
field='3C286,3C295'; field named 3C286 and 3C295
field = '3,4C*'; field id 3, all names starting with 4C
spw --Select spectral window/channels
NOTE: This selects the data passed as the INPUT to mode
default: ''=all spectral windows and channels
spw='0~2,4'; spectral windows 0,1,2,4 (all channels)
spw='0:5~61'; spw 0, channels 5 to 61
spw='<2'; spectral windows less than 2 (i.e. 0,1)
spw='0,10,3:3~45'; spw 0,10 all channels, spw 3,
channels 3 to 45.
spw='0~2:2~6'; spw 0,1,2 with channels 2 through 6 in each.
spw='0:0~10;15~60'; spectral window 0 with channels
0-10,15-60
spw='0:0~10,1:20~30,2:1;2;3'; spw 0, channels 0-10,
spw 1, channels 20-30, and spw 2, channels, 1,2 and 3
selectdata -- Other data selection parameters
default: True
selectdata=True expandable parameters
antenna -- Select data based on antenna/baseline
default: '' (all)
If antenna string is a non-negative integer, it is
assumed to be an antenna index, otherwise, it is
considered an antenna name.
antenna='5&6'; baseline between antenna index 5 and
index 6.
antenna='VA05&VA06'; baseline between VLA antenna 5
and 6.
antenna='5&6;7&8'; baselines 5-6 and 7-8
antenna='5'; all baselines with antenna index 5
antenna='05'; all baselines with antenna number 05
(VLA old name)
antenna='5,6,9'; all baselines with antennas 5,6,9
index numbers
timerange -- Select data based on time range:
default = '' (all); examples,
timerange = 'YYYY/MM/DD/hh:mm:ss~YYYY/MM/DD/hh:mm:ss'
Note: if YYYY/MM/DD is missing date defaults to first
day in data set
timerange='09:14:0~09:54:0' picks 40 min on first day
timerange= '25:00:00~27:30:00' picks 1 hr to 3 hr
30min on NEXT day
timerange='09:44:00' pick data within one integration
of time
timerange='>10:24:00' data after this time
scan -- Scan number range.
default: '' (all)
example: scan='1~5'
Check 'go listobs' to insure the scan numbers are in
order.
array -- Select data by (sub)array indices
default: '' (all); example:
array='0~2'; arrays 0 to 2
mode -- Frequency Specification:
NOTE: See examples below:
default: 'channel'
mode = 'channel'; Use with nchan, start, width to specify
output spw. See examples below
mode = 'velocity', means channels are specified in
velocity.
mode = 'frequency', means channels are specified in
frequency.
mode expandable parameters
Start, width are given in units of channels, frequency
or velocity as indicated by mode
nchan -- Number of channels in output spw
default: -1 = all channels; example: nchan=3
start -- Start input channel (relative-0)
default=0; example: start=5
width -- Output channel width in units of the input
channel width (>1 indicates channel averaging)
default=1; example: width=4
interpolation -- Interpolation method
default = 'linear'
examples:
spw = '0,1'; mode = 'channel'
will produce a single spw containing all channels in spw
0 and 1
spw='0:5~28^2'; mode = 'channel'
will produce a single spw made with channels
(5,7,9,...,25,27)
spw = '0'; mode = 'channel': nchan=3; start=5; width=4
will produce an spw with 3 output channels
new channel 1 contains data from channels (5+6+7+8)
new channel 2 contains data from channels (9+10+11+12)
new channel 3 contains data from channels (13+14+15+16)
spw = '0:0~63^3'; mode='channel'; nchan=21; start = 0;
width = 1
will produce an spw with 21 channels
new channel 1 contains data from channel 0
new channel 2 contains data from channel 2
new channel 21 contains data from channel 61
spw = '0:0~40^2'; mode = 'channel'; nchan = 3; start =
5; width = 4
will produce an spw with three output channels
new channel 1 contains channels (5,7)
new channel 2 contains channels (13,15)
new channel 3 contains channels (21,23)
phasecenter -- direction measure or fieldid for the mosaic center
default: '' => first field selected ; example: phasecenter=6
or phasecenter='J2000 19h30m00 -40d00m00'
restfreq -- Specify rest frequency to use for output image
default='' Occasionally it is necessary to set this (for
example some VLA spectral line data). For example for
NH_3 (1,1) put restfreq='23.694496GHz'
outframe -- output reference frame (not case-sensitive)
possible values: LSRK, LSRD, BARY, GALACTO, LGROUP, CMB, GEO, TOPO or SOURCE
default='' (keep original reference frame) ; example: outframe='BARY'
veltype -- definition of velocity (in mode)
default = 'radio'
hanning -- if true, Hanning smooth frequency channel data before regridding
to remove Gibbs ringing
default = False
"""
#Python script
# make ms and tb tool local
ms = casac.ms()
tb = casac.table()
try:
casalog.origin('cvel')
if not ((type(vis) == str) & (os.path.exists(vis))):
raise Exception, 'Visibility data set not found - please verify the name'
if (outputvis == ""):
raise Exception, "Must provide output data set name in parameter outputvis."
if os.path.exists(outputvis):
raise Exception, "Output MS %s already exists - will not overwrite." % outputvis
if (os.path.exists(outputvis + ".flagversions")):
raise Exception, "The flagversions \"%s.flagversions\" for the output MS already exist. Please delete." % outputvis
# Handle selectdata explicitly
# (avoid hidden globals)
if (selectdata == False):
timerange = ''
array = ''
antenna = ''
scan = ''
if (type(antenna) == list):
antenna = ', '.join([str(ant) for ant in antenna])
if (field == ''):
field = '*'
if (spw == ''):
spw = '*'
if (passall and spw == '*' and field == '*'):
# all spws and fields selected, nothing to pass through
passall = False
doselection = True
if (field == '*' and spw == '*'
and (not selectdata or
(selectdata and antenna == '' and timerange == ''
and scan == '' and array == ''))):
doselection = False
# open input MS
ms.open(vis)
if (type(phasecenter) == str):
### blank means take field 0
if (phasecenter == ''):
phasecenter = ms.msseltoindex(vis, field=field)['field'][0]
else:
tmppc = phasecenter
try:
if (len(ms.msseltoindex(vis, field=phasecenter)['field']) >
0):
tmppc = ms.msseltoindex(vis,
field=phasecenter)['field'][0]
##succesful must be string like '0' or 'NGC*'
except Exception, instance:
##failed must be a string 'J2000 18h00m00 10d00m00'
tmppc = phasecenter
phasecenter = tmppc
newphasecenter = phasecenter
phasecentername = phasecenter
if not (type(phasecenter) == str):
# first check that this field will be in the output MS
if not (phasecenter in ms.msseltoindex(vis, field=field)['field']):
message = "Field id " + str(phasecenter)
message += " was selected as phasecenter but is not among the fields selected for output: "
message += str(ms.msseltoindex(vis, field=field)['field'])
ms.close()
raise Exception, message
tb.open(vis + "/FIELD")
try:
# get the name for later display
phasecentername = tb.getcell(
'NAME',
phasecenter) + " (original field " + str(phasecenter)
tb.close()
# phase center id will change if we select on fields,
# the name column is only optionally filled and not necessarily unique.
# But ms.msseltoindex(vis,field=field)['field'] returns the old field ids
# in the order in which they will occur in the new field table.
# => need to get index from there as new phase center ID
newphasecenter = (ms.msseltoindex(
vis, field=field)['field']).tolist().index(phasecenter)
phasecentername += ", new field " + str(newphasecenter) + ")"
except:
tb.close()
message = "phasecenter field id " + str(
phasecenter) + " is not valid"
ms.close()
raise Exception, message
if (mode == 'frequency'):
## reset the default values
if (start == 0):
start = ""
if (width == 1):
width = ""
##check that start and width have units if they are non-empty
if not (start == ""):
if (qa.quantity(start)['unit'].find('Hz') < 0):
ms.close()
raise TypeError, "start parameter is not a valid frequency quantity " % start
if (not (width == "")
and (qa.quantity(width)['unit'].find('Hz') < 0)):
ms.close()
raise TypeError, "width parameter %s is not a valid frequency quantity " % width
elif (mode == 'velocity'):
## reset the default values
if (start == 0):
start = ""
if (width == 1):
width = ""
##check that start and width have units if they are non-empty
if not (start == ""):
if (qa.quantity(start)['unit'].find('m/s') < 0):
ms.close()
raise TypeError, "start parameter %s is not a valid velocity quantity " % start
if (not (width == "")
and (qa.quantity(width)['unit'].find('m/s') < 0)):
ms.close()
raise TypeError, "width parameter %s is not a valid velocity quantity " % width
elif (mode == 'channel' or mode == 'channel_b'):
if ((type(width) != int) or (type(start) != int)):
ms.close()
raise TypeError, "start and width have to be integers with mode = %s" % mode
## check if preaveraging is necessary
dopreaverage = False
preavwidth = [1]
thespwsel = ms.msseltoindex(vis=vis, spw=spw)['spw']
thefieldsel = ms.msseltoindex(vis=vis, field=field)['field']
outgrid = ms.cvelfreqs(spwids=thespwsel,
fieldids=thefieldsel,
mode=mode,
nchan=nchan,
start=start,
width=width,
phasec=newphasecenter,
restfreq=restfreq,
outframe=outframe,
veltype=veltype,
verbose=False)
if (len(outgrid) > 1):
tmpavwidth = []
for thespw in thespwsel:
outgridw1 = ms.cvelfreqs(
spwids=[thespw],
fieldids=thefieldsel,
mode='channel',
nchan=-1,
start=0,
width=1, # native width
phasec=newphasecenter,
restfreq=restfreq,
outframe=outframe,
veltype=veltype,
verbose=False)
if (len(outgridw1) > 1):
widthratio = abs((outgrid[1] - outgrid[0]) /
(outgridw1[1] - outgridw1[0]))
if (widthratio >= 2.0): # do preaverage
tmpavwidth.append(int(widthratio + 0.001))
dopreaverage = True
else:
tmpavwidth.append(1)
if dopreaverage:
preavwidth = tmpavwidth
ms.close()
# if in channel mode and preaveraging,
if (dopreaverage and (mode == 'channel' or mode == 'channel_b')):
if (max(preavwidth) == 1):
dopreaverage = False
else:
# switch to frequency mode
mode = 'frequency'
if (width > 0):
start = str(outgrid[0] / 1E6) + 'MHz'
width = str((outgrid[1] - outgrid[0])) + 'Hz'
else:
start = str(outgrid[len(outgrid) - 1] / 1E6) + 'MHz'
width = str(-(outgrid[1] - outgrid[0])) + 'Hz'
casalog.post(
"After pre-averaging, will switch to frequency mode with start="
+ start + ", width = " + width, 'INFO')
if (dopreaverage and hanning and max(preavwidth) > 2):
casalog.post(
"NOTE: You have requested Hanning smoothing and at the same time you have chosen\n"
+
"a large width parameter which makes pre-averaging necessary.\n"
+ "The Hanning-smoothing may be redundant in this context.\n",
'WARN')
if dopreaverage:
# Past this point we know we are going to 'dopreaverage'
# CAS-9798
raise RuntimeError(
'ERROR: cvel does not regrid properly for channel '
'widths > or = 2x the native channel width, please use '
'mstransform, clean, or tclean for larger regridding. '
'All earlier versions of CASA also have this issue.')
# determine parameter "datacolumn"
tb.open(vis)
allcols = tb.colnames()
tb.close()
dpresent = ('DATA' in allcols)
mpresent = ('MODEL_DATA' in allcols)
cpresent = ('CORRECTED_DATA' in allcols)
if (dpresent and cpresent):
datacolumn = 'all'
elif (dpresent and not cpresent):
datacolumn = 'data'
elif (cpresent and not dpresent):
datacolumn = 'corrected_data'
elif (mpresent and not cpresent and not dpresent):
datacolumn = 'model_data'
else:
raise Exception, "Neither DATA nor CORRECTED_DATA nor MODEL_DATA column present. Cannot proceed."
if (doselection and not dopreaverage):
casalog.post("Creating selected SubMS ...", 'INFO')
ms.open(vis)
ms.split(outputms=outputvis,
field=field,
spw=spw,
step=[1],
baseline=antenna,
subarray=array,
timebin='-1s',
time=timerange,
whichcol=datacolumn,
scan=scan,
uvrange="")
ms.close()
elif (dopreaverage and not doselection):
if (hanning):
casalog.post("Creating working copy for Hanning-smoothing ...",
'INFO')
shutil.rmtree(outputvis + 'TMP', ignore_errors=True)
shutil.copytree(vis, outputvis + 'TMP')
ms.open(outputvis + 'TMP', nomodify=False)
ms.hanningsmooth(datacolumn=datacolumn)
ms.close()
hanning = False
ms.open(outputvis + 'TMP')
else:
ms.open(vis)
casalog.post(
"Creating preaveraged SubMS using widths " + str(preavwidth),
'INFO')
ms.split(outputms=outputvis, whichcol=datacolumn, step=preavwidth)
ms.close()
elif (doselection and dopreaverage):
if (hanning):
casalog.post(
"Creating selected working copy for Hanning-smoothing ...",
'INFO')
shutil.rmtree(outputvis + 'TMP', ignore_errors=True)
ms.open(vis)
ms.split(outputms=outputvis + 'TMP',
field=field,
spw=spw,
step=[1],
baseline=antenna,
subarray=array,
timebin='-1s',
time=timerange,
whichcol=datacolumn,
scan=scan,
uvrange="")
ms.close()
ms.open(outputvis + 'TMP', nomodify=False)
ms.hanningsmooth(datacolumn=datacolumn)
ms.close()
hanning = False
ms.open(outputvis + 'TMP')
casalog.post(
"Creating preaveraged SubMS using widths " +
str(preavwidth), 'INFO')
ms.split(outputms=outputvis,
whichcol=datacolumn,
step=preavwidth)
ms.close()
else:
casalog.post(
"Creating selected, preaveraged SubMS using widths " +
str(preavwidth), 'INFO')
ms.open(vis)
ms.split(outputms=outputvis,
field=field,
spw=spw,
step=preavwidth,
baseline=antenna,
subarray=array,
timebin='-1s',
time=timerange,
whichcol=datacolumn,
scan=scan,
uvrange="")
ms.close()
else:
# no selection or preaveraging necessary, just copy
casalog.post("Creating working copy ...", 'INFO')
shutil.rmtree(outputvis, ignore_errors=True)
shutil.copytree(vis, outputvis)
# Combine and if necessary regrid it
ms.open(outputvis, nomodify=False)
message = "Using " + phasecentername + " as common direction for the output reference frame."
casalog.post(message, 'INFO')
if not ms.cvel(mode=mode,
nchan=nchan,
start=start,
width=width,
interp=interpolation,
phasec=newphasecenter,
restfreq=restfreq,
outframe=outframe,
veltype=veltype,
hanning=hanning):
ms.close()
raise Exception, "Error in regridding step ..."
ms.close()
# deal with the passall option
temp_suffix = ".deselected"
if (passall):
# determine range of fields
fieldsel = ms.msseltoindex(vis=vis, field=field)['field']
tb.open(vis + "/FIELD")
nfields = tb.nrows()
tb.close()
fielddesel = ""
for i in xrange(0, nfields):
if not (i in fieldsel):
if not (fielddesel == ""):
fielddesel += ","
fielddesel += str(i)
# determine range of SPWs
spwsel = ms.msseltoindex(vis=vis, spw=spw)['spw']
tb.open(vis + "/SPECTRAL_WINDOW")
nspws = tb.nrows()
tb.close()
spwdesel = ""
for i in xrange(0, nspws):
if not (i in spwsel):
if not (spwdesel == ""):
spwdesel += ","
spwdesel += str(i)
if not (fielddesel == "" and spwdesel == ""):
# split out the data not selected by the conditions on field and spw
# from the original MS and join it to the output MS
# need to do this in two steps
# I) field == "*" and deselected spws
if not (spwdesel == ""):
ms.open(vis)
casalog.post("Passing through data with", 'INFO')
casalog.post(" spws: " + spwdesel, 'INFO')
ms.split(outputms=outputvis + temp_suffix,
field='*',
spw=spwdesel,
step=[1],
baseline=antenna,
subarray=array,
timebin='-1s',
time=timerange,
whichcol=datacolumn,
scan=scan,
uvrange="")
ms.close()
# join with the deselected part
ms.open(outputvis, nomodify=False)
rval = ms.concatenate(msfile=outputvis + temp_suffix)
ms.close()
shutil.rmtree(outputvis + temp_suffix)
if not rval:
raise Exception, "Error in attaching passed-through data ..."
# II) deselected fields and selected spws
if not (fielddesel == ""):
ms.open(vis)
casalog.post("Passing through data with", 'INFO')
casalog.post(" fields: " + fielddesel, 'INFO')
casalog.post(" spws: " + spw, 'INFO')
ms.split(outputms=outputvis + temp_suffix,
field=fielddesel,
spw=spw,
step=[1],
baseline=antenna,
subarray=array,
timebin='-1s',
time=timerange,
whichcol=datacolumn,
scan=scan,
uvrange="")
ms.close()
# join with the deselected part
ms.open(outputvis, nomodify=False)
rval = ms.concatenate(msfile=outputvis + temp_suffix)
ms.close()
shutil.rmtree(outputvis + temp_suffix)
if not rval:
raise Exception, "Error in attaching passed-through data ..."
# Write history to output MS
try:
param_names = cvel.func_code.co_varnames[:cvel.func_code.
co_argcount]
param_vals = [eval(p) for p in param_names]
write_history(mstool(), outputvis, 'cvel', param_names, param_vals,
casalog)
except Exception, instance:
casalog.post("*** Error \'%s\' updating HISTORY" % (instance),
'WARN')
def sdbaseline(infile=None, datacolumn=None, antenna=None, field=None,
spw=None, timerange=None, scan=None, pol=None, intent=None,
reindex=None, maskmode=None, thresh=None, avg_limit=None,
minwidth=None, edge=None, blmode=None, dosubtract=None,
blformat=None, bloutput=None, bltable=None, blfunc=None,
order=None, npiece=None, applyfft=None, fftmethod=None,
fftthresh=None, addwn=None, rejwn=None, clipthresh=None,
clipniter=None, blparam=None, verbose=None, showprogress=None,
minnrow=None, outfile=None, overwrite=None):
casalog.origin('sdbaseline')
try:
if not os.path.exists(infile):
raise Exception("infile='" + str(infile) + "' does not exist.")
if (outfile == '') or not isinstance(outfile, str):
#print("type=%s, value=%s" % (type(outfile), str(outfile)))
#raise ValueError, "outfile name is empty."
outfile = infile.rstrip('/') + '_bs'
print("outfile is empty or non-string. set to '" + outfile + "'")
if os.path.exists(outfile) and not overwrite:
raise Exception("outfile='%s' exists, and cannot overwrite it." % (outfile))
if (maskmode == 'interact'):
raise ValueError, "maskmode='%s' is not supported yet" % maskmode
if (blfunc == 'variable' and not os.path.exists(blparam)):
raise ValueError, "input file '%s' does not exists" % blparam
if (spw == ''): spw = '*'
if (blmode == 'apply'):
if not os.path.exists(bltable):
raise ValueError, "file specified in bltable '%s' does not exist." % bltable
sorttab_info = remove_sorted_table_keyword(infile)
if overwrite and os.path.exists(outfile) and (infile != outfile):
os.system('rm -rf %s' % outfile)
selection = ms.msseltoindex(vis=infile, spw=spw, field=field,
baseline=antenna, time=timerange,
scan=scan)
sdms.open(infile)
sdms.set_selection(spw=sdutil.get_spwids(selection), field=field,
antenna=antenna, timerange=timerange,
scan=scan, polarization=pol, intent=intent,
reindex=reindex)
sdms.apply_baseline_table(bltable=bltable,
datacolumn=datacolumn,
spw=spw,
outfile=outfile)
sdms.close()
restore_sorted_table_keyword(infile, sorttab_info)
elif (blmode == 'fit'):
if(blfunc == 'sinusoid'):
addwn = sdutil.parse_wavenumber_param(addwn)
rejwn = sdutil.parse_wavenumber_param(rejwn)
check_fftthresh(fftthresh)
blformat, bloutput = prepare_for_blformat_bloutput(infile, blformat, bloutput, overwrite)
output_bloutput_text_header(blformat, bloutput,
blfunc, maskmode,
infile, outfile)
if (blfunc == 'variable'):
sorttab_info = remove_sorted_table_keyword(infile)
if overwrite and os.path.exists(outfile) and (infile != outfile):
os.system('rm -rf %s' % outfile)
selection = ms.msseltoindex(vis=infile, spw=spw, field=field,
baseline=antenna, time=timerange,
scan=scan)
sdms.open(infile)
sdms.set_selection(spw=sdutil.get_spwids(selection),
field=field, antenna=antenna,
timerange=timerange, scan=scan,
polarization=pol, intent=intent,
reindex=reindex)
params, func = prepare_for_baselining(blfunc=blfunc,
datacolumn=datacolumn,
outfile=outfile,
bloutput=','.join(bloutput),
dosubtract=dosubtract,
spw=spw,
pol=pol,
linefinding=(maskmode=='auto'),
threshold=thresh,
avg_limit=avg_limit,
minwidth=minwidth,
edge=edge,
order=order,
npiece=npiece,
applyfft=applyfft,
fftmethod=fftmethod,
fftthresh=fftthresh,
addwn=addwn,
rejwn=rejwn,
clip_threshold_sigma=clipthresh,
num_fitting_max=clipniter+1,
blparam=blparam,
verbose=verbose)
func(**params)
sdms.close()
if (blfunc == 'variable'):
restore_sorted_table_keyword(infile, sorttab_info)
# Write history to outfile
param_names = sdbaseline.func_code.co_varnames[:sdbaseline.func_code.co_argcount]
param_vals = [eval(p) for p in param_names]
write_history(ms, outfile, 'sdbaseline', param_names,
param_vals, casalog)
except Exception, instance:
raise Exception, instance
fh.writeFlagCommands(vis, flagcmd, writeflags, cmdreason,
outfile, False)
# Destroy the tool
aflocal.done()
retval = True
# Write history to the MS. Only for modes that write to the MS
if not iscal:
if mode != 'summary' and action == 'apply':
try:
param_names = flagdata.func_code.co_varnames[:flagdata.
func_code.
co_argcount]
param_vals = [eval(p) for p in param_names]
retval &= write_history(mslocal, vis, 'flagdata',
param_names, param_vals, casalog)
except Exception, instance:
casalog.post(
"*** Error \'%s\' updating HISTORY" % (instance),
'WARN')
# Pull out the 'summary' reports of summary_stats_list.
if mode == 'summary' or mode == 'list':
ordered_summary_list = OrderedDict(summary_stats_list)
nreps = ordered_summary_list['nreport']
if nreps > 0:
for rep in range(0, nreps):
repname = "report" + str(rep)
if ordered_summary_list[repname]['type'] != "summary":
ordered_summary_list.pop(repname)
def importasap(infile=None,
outputvis=None,
flagbackup=None,
overwrite=None,
parallel=None):
"""
"""
casalog.origin('importasap')
try:
if infile is None or outputvis is None:
raise RuntimeError(
'Error: infile and outputvis must be specified.')
# default value
if flagbackup is None:
flagbackup = True
if overwrite is None:
overwrite = False
# basic check
if os.path.exists(outputvis) and not overwrite:
raise RuntimeError('%s exists.' % (outputvis))
if not is_scantable(infile):
raise RuntimeError('%s is not a valid Scantable.' % (infile))
# import
status = mysdms.importasap(infile, outputvis, parallel)
if status == True:
# flagversions file must be deleted
flagversions = outputvis.rstrip('/') + '.flagversions'
if os.path.exists(flagversions):
os.system('rm -rf %s' % (flagversions))
# initialize weights using cb tool
mycb.open(outputvis, compress=False, addcorr=False, addmodel=False)
mycb.initweights(wtmode='nyq')
# create flagbackup file if user requests it
if flagbackup == True:
aflocal = casac.agentflagger()
aflocal.open(outputvis)
aflocal.saveflagversion(
'Original',
comment=
'Original flags at import into CASA using importasap',
merge='save')
aflocal.done()
# Write history to output MS
param_names = importasap.func_code.co_varnames[:importasap.func_code.
co_argcount]
param_vals = [eval(p) for p in param_names]
write_history(myms, outputvis, 'importasap', param_names, param_vals,
casalog)
return status
except Exception, instance:
casalog.post('*** Error *** ' + str(instance), 'SEVERE')
raise instance
raise Exception, 'Tsys calibration table %s not found' % tsystable
if len(spwmap)==0:
spwmap=[-1]
if interp=="":
interp="linear"
# ... and we are asked to do something...
# open without adding anything!
mycb.open(vis,compress=False,addcorr=False,addmodel=False)
mycb.initweights(wtmode=wtmode,dowtsp=dowtsp,tsystable=tsystable,gainfield=gainfield,interp=interp,spwmap=spwmap)
mycb.close()
else:
raise Exception, 'Visibility data set not found - please verify the name'
# Write history to MS.
# When running in parallel, history will be written in the parallel section above
# normal MSs should write the history here
if ParallelTaskHelper.isMPIClient():
try:
param_names = initweights.func_code.co_varnames[:initweights.func_code.co_argcount]
param_vals = [eval(p) for p in param_names]
casalog.post('Updating the history in the output', 'DEBUG1')
write_history(myms, vis, 'initweights', param_names,
param_vals, casalog)
except Exception, instance:
casalog.post("*** Error \'%s\' updating HISTORY" % (instance),
'WARN')
except Exception, instance:
print '*** Error ***',instance
def sdcal(infile=None, calmode='tsys', fraction='10%', noff=-1,
width=0.5, elongated=False, applytable='',interp='', spwmap={},
outfile='', overwrite=False, field='', spw='', scan='',intent=''):
""" Externally specify calibration solutions of various types
"""
# Composite mode: compute calibration table and calibrate
if ',' in calmode:
handle_composite_mode(locals())
return
# Single mode: either calibrate or compute calibration table
try:
# Parameters check
if calmode == 'tsys':
if scan != '':
raise UserWarning, "Scan input must be ''(=all) in calmode='tsys'."
if spw != '':
raise UserWarning, "Spw input must be ''(=all) in calmode='tsys'."
if isinstance(infile,str) and os.path.exists(infile):
# check if CORRECTED_DATA is necessary
addcorr = calmode == 'apply'
cb.setvi(old=True)
cb.open(filename=infile,compress=False,addcorr=addcorr,addmodel=False)
cb.selectvis(spw=spw, scan=scan, field=field)
else:
raise Exception, 'Infile data set not found - please verify the name'
if not isinstance(calmode,str):
raise Exception, "Calmode must be a string"
if calmode.lower() not in ['tsys', 'ps', 'otfraster', 'otf', 'apply']:
raise Exception, "Calmode must be either 'ps' or 'otfraster' or 'otf' or 'tsys' or 'apply'."
if (not overwrite) and os.path.exists(outfile):
raise RuntimeError, "overwrite is False and output file exists: {}".format(outfile)
# Calibration
if calmode == 'apply': # Calibrate using existing tables
# single calibration table
if isinstance(applytable, str):
_table_list = [applytable]
# multiple calibration tables
if isinstance(applytable, list) or isinstance(applytable, numpy.ndarray):
_table_list = applytable
# no calibration table
if len(_table_list) == 0:
raise Exception, 'Applytable name must be specified.'
# check calibration table files
for table in _table_list:
# empty string
if len(table) == 0:
raise Exception, 'Applytable name must be specified.'
# unexisting table
if not os.path.exists(table):
raise Exception, "Table doesn't exist: {}".format(table)
# warning on usage difference with asap.sdcal2
if (outfile != ''):
warning_msg = '\n'.join([
'The outfile you specified will NOT be created.',
"Calibrated data will be stored in a new 'CORRECTED_DATA' column",
'inserted in the main table of the input MS file.'
])
casalog.post(warning_msg,priority="WARN")
if(type(spwmap)!=types.ListType and (type(spwmap)!=types.DictType)):
raise Exception, 'Spwmap type must be list or dictionary.'
if (type(spwmap)==types.DictType):
MS = infile
tb.open(MS+'/SPECTRAL_WINDOW')
total_spwID=tb.nrows()
tb.close()
spwmap_dict = spwmap
spwmap_list = range(total_spwID)
for key, value in spwmap_dict.items():
for v in value:
if v in spwmap_list:
index = spwmap_list.index(v)
spwmap_list[index]=int(key)
spwmap = spwmap_list
# Setup calibrator
for _table in _table_list:
caltype = inspect_caltype(_table)
if caltype == 'UNKNOWN':
raise RuntimeError('Applytable \'%s\' is not a caltable format'%(_table))
elif caltype == 'B TSYS':
cb.setapply(table=_table, spwmap=spwmap, interp=interp, calwt=True)
else:
# no spw mapping is needed for sky calibration
cb.setapply(table=_table, interp=interp, calwt=True)
# Calibrate
cb.correct(applymode='calflag')
# Write to HISTORY table of MS
param_names = sdcal.func_code.co_varnames[:sdcal.func_code.co_argcount]
param_vals = [eval(p) for p in param_names]
write_history(myms, infile, 'sdcal', param_names,
param_vals, casalog)
else: # Compute calibration table
# Reconciliating 'Python world' calmode with 'C++ world' calmode
# cpp_calmode[python_calmode] = c++_calmode
cpp_calmode = { 'tsys': 'tsys',
'ps': 'sdsky_ps',
'otfraster': 'sdsky_raster',
'otf': 'sdsky_otf'
}
if len(outfile) == 0:
raise RuntimeError, 'Output file name must be specified.'
if calmode == 'tsys':
cb.specifycal(caltable=outfile,time="",spw=spw,caltype=cpp_calmode[calmode])
else:
fraction_numeric = to_numeric_fraction(fraction)
if noff <= 0 and fraction_numeric >= 0.5:
raise ValueError, 'Too many edge points. fraction must be < 0.5.'
# Setup calibrator
cb.selectvis(spw=spw, scan=scan, field=field, intent=intent)
cb.setsolve(type=cpp_calmode[calmode], table=outfile, fraction=fraction_numeric, numedge=noff)
# Compute calibration table
cb.solve()
except UserWarning, instance:
print '*** Warning ***',instance
shutil.rmtree(csvis)
return False
else:
# This takes almost 30s/GB. (lustre, 8/2011)
casalog.post('Copying ' + vis + ' to ' + csvis + ' with cp.')
copy_tree(vis, csvis)
# It is less confusing if we write the history now that the "root" MS
# is made, but before cb adds its messages.
#
# Not a dict, because we want to maintain the order.
param_names = uvcontsub.func_code.co_varnames[:uvcontsub.func_code.
co_argcount]
param_vals = [eval(p) for p in param_names]
write_history(myms, csvis, 'uvcontsub', param_names, param_vals,
casalog)
if (type(csvis) == str) and os.path.isdir(csvis):
mycb.setvi(old=True, quiet=False)
# old VI, for now
mycb.open(csvis)
else:
raise Exception, 'Visibility data set not found - please verify the name'
# select the data for continuum subtraction
mycb.reset()
mycb.selectvis(spw=myfitspw)
# Set up the solve
amuellertab = tempfile.mkdtemp(prefix='Temp_contsub.tab',
dir=workingdir)
casalog.post(
'FLAG_CMD table contains spw selection by name. Will not update it!',
'DEBUG')
mytb.close()
except Exception, instance:
if isopen:
mytb.close()
mslocal = None
mytb = None
casalog.post("*** Error \'%s\' updating FLAG_CMD" % (instance),
'SEVERE')
return False
mytb = None
# Write history to output MS, not the input ms.
try:
param_names = split.func_code.co_varnames[:split.func_code.co_argcount]
param_vals = [eval(p) for p in param_names]
write_history(mslocal, outputvis, 'split', param_names, param_vals,
casalog)
except Exception, instance:
casalog.post("*** Error \'%s\' updating HISTORY" % (instance), 'WARN')
return False
mslocal = None
return True
def conjugatevis(vis, spwlist=[], outputvis="", overwrite=False):
""":
Change the sign of the phases in all visibility columns
Keyword arguments:
vis -- Name of input visibility file
default: none; example='3C273XC1.ms'
spwlist -- Select spectral window
default: [] all spws will be conjugated; example: spw=[1,2]
outputvis -- name of output visibility file
default: 'conjugated_'+vis; example= 'conjugated.ms'
overwrite -- Overwrite the outputvis if it exists
default=False; example: overwrite=True
"""
#Python script
tb = tbtool()
try:
casalog.origin('conjugatevis')
myddlist = []
tb.open(vis + '/SPECTRAL_WINDOW')
maxspw = tb.nrows() - 1
tb.close()
if (type(spwlist) == type(1)):
spwlist = [spwlist]
elif (spwlist == None or spwlist == [] or spwlist == ""):
spwlist = []
casalog.post("Will conjugate visibilities for all spws.", 'INFO')
if not spwlist == []:
try:
tb.open(vis + '/DATA_DESCRIPTION')
for k in spwlist:
if (k < -1 or k > maxspw):
raise Exception, "Error: max valid spw id is " + str(
maxspw)
raise
else:
for j in range(0, tb.nrows()):
if (tb.getcell("SPECTRAL_WINDOW_ID", j) == k
and not (j in myddlist)):
myddlist = myddlist + [j]
#end for k
tb.close()
casalog.post('DATA_DESC_IDs to process: ' + str(myddlist),
'INFO')
except:
raise Exception, 'Error reading DATA_DESCRIPTION table'
#endif
outname = 'conjugatedvis_' + vis
if not outputvis == "":
if ((type(outputvis) != str) or (len(outputvis.split()) < 1)):
raise Exception, 'parameter outputvis is invalid'
outname = outputvis
if not overwrite and os.path.exists(outname):
raise Exception, 'outputvis ' + outname + ' exists and you did not permit overwrite'
os.system('rm -rf ' + outname)
os.system('cp -R ' + vis + ' ' + outname)
tb.open(outname, nomodify=False)
if tb.iswritable():
if (spwlist == None):
for colname in ['DATA', 'CORRECTED_DATA', 'FLOAT_DATA']:
if colname in tb.colnames():
casalog.post('Conjugating ' + str(colname), 'INFO')
for i in xrange(0, tb.nrows()):
a = tb.getcell(colname, i)
a = a.conjugate()
tb.putcell(colname, i, a)
else:
for colname in ['DATA', 'CORRECTED_DATA', 'FLOAT_DATA']:
if colname in tb.colnames():
casalog.post('Conjugating ' + str(colname), 'INFO')
for i in xrange(0, tb.nrows()):
if (tb.getcell("DATA_DESC_ID", i) in myddlist):
a = tb.getcell(colname, i)
a = a.conjugate()
tb.putcell(colname, i, a)
#endif
tb.flush()
tb.close()
casalog.post('Created ' + str(outname), 'INFO')
else:
tb.close()
casalog.post('Cannot write to output MS ' + str(outname), 'WARN')
# Write history to output MS
try:
param_names = conjugatevis.func_code.co_varnames[:conjugatevis.
func_code.
co_argcount]
param_vals = [eval(p) for p in param_names]
write_history(mstool(), outputvis, 'conjugatevis', param_names,
param_vals, casalog)
except Exception, instance:
casalog.post("*** Error \'%s\' updating HISTORY" % (instance),
'WARN')
except Exception, instance:
tb.close()
print '*** Error ***', instance
raise Exception, instance
def importmiriad(
mirfile=None,
vis=None,
tsys=None,
spw=None,
vel=None,
linecal=None,
wide=None,
debug=None,
):
"""Convert a Miriad visibility file into a CASA visibility file (MS).
The conversion of the Miriad visibility format into a measurement set. This version
has been tested for both ATNF and CARMA Miriad files.
................
Keyword arguments:
mirfile -- Name of input Miriad visibility file (directory)
default: none; example: mirfile='mydata.uv'
.... vis -- Output ms name
default: mirfile name with suffix replaced by '.ms'
.... tsys -- Set True to use the system temperature to set the visibility weights
default: False
.... spw -- specify the spectral windows to use
........ default: all
.... vel -- Velocity system to use: LSRK, LSRD or TOPO
.... default: TOPO for ATCA, LSRK for CARMA
.... linecal -- (CARMA only) apply CARMA linecal on the fly?
.... default: False
.... wide -- (CARMA only) specify the window averages to use
........ default: all
........
.... debug -- specify level of debug messages (0,1,2,3)
default: 0 (=None)
"""
# Python script
mymf = casac.miriadfiller()
try:
try:
casalog.origin('importmiriad')
# -----------------------------------------'
# beginning of importmiriad implementation
# -----------------------------------------
mymf.fill(vis, mirfile, tsys, spw, vel, linecal, wide, debug)
except Exception, e:
print e
casalog.post("Failed to import miriad file %s" % mirfile)
raise
# Write the args to HISTORY.
try:
mslocal = mstool()
param_names = \
importmiriad.func_code.co_varnames[:importmiriad.func_code.co_argcount]
param_vals = [eval(p) for p in param_names]
write_history(mslocal, vis, 'importmiriad', param_names,
param_vals, casalog)
except Exception, instance:
casalog.post("Failed to updated HISTORY", 'WARN')
antnamescheme=antnamescheme,
keepblanks=keepblanks,
evlabands=evlabands)
i += 1
else:
raise Exception, 'Archive file not found - please verify the name'
except Exception, instance:
print '*** Error importing %s to %s:' % (archivefiles, vis)
raise Exception, instance
# Write history
try:
param_names = importvla.func_code.co_varnames[:importvla.func_code.
co_argcount]
param_vals = [eval(p) for p in param_names]
ok &= write_history(mstool(), vis, 'importvla', param_names,
param_vals, casalog)
except Exception, instance:
casalog.post("*** Error \'%s\' updating HISTORY" % (instance), 'WARN')
# write initial flag version
try:
aflocal = casac.agentflagger()
ok &= aflocal.open(vis)
ok &= aflocal.saveflagversion(
'Original',
comment='Original flags at import into CASA',
merge='replace')
ok &= aflocal.done()
except Exception, instance:
print '*** Error writing initial flag version of %s:' % vis
raise Exception, instance
mytb.close()
except Exception, instance:
if isopen:
mytb.close()
mslocal = None
mytb = None
casalog.post("*** Error \'%s\' updating FLAG_CMD" % (instance),
'SEVERE')
return False
mytb = None
# Write history to output MS, not the input ms.
try:
param_names = mstransform.func_code.co_varnames[:mstransform.func_code.co_argcount]
param_vals = [eval(p) for p in param_names]
write_history(mslocal, outputvis, 'mstransform', param_names,
param_vals, casalog)
except Exception, instance:
casalog.post("*** Error \'%s\' updating HISTORY" % (instance),
'WARN')
return False
mslocal = None
return True
def virtualconcat(vislist, concatvis, freqtol, dirtol, respectname,
visweightscale, keepcopy, copypointing):
"""
Concatenate visibility data sets creating a Multi-MS.
Combine the input datasets into a Multi-MS.
NOTE: The input datasets are moved into the Multi-MS and may be modified
to account for subtable reference changes.
If none of the input MSs have any scratch columns, none are created.
Otherwise scratch columns are created and initialized in those MSs
which don't have a complete set.
Keyword arguments:
vis -- Name of input visibility files (MS)
default: none; example: vis=['ngc5921-1.ms', 'ngc5921-2.ms']
concatvis -- Name of the output visibility file
default: none; example: concatvis='src2.ms'
freqtol -- Frequency shift tolerance for considering data as the same spwid
default: '' means always combine
example: freqtol='10MHz' will not combine spwid unless they are
within 10 MHz
dirtol -- Direction shift tolerance for considering data as the same field
default: ;; means always combine
example: dirtol='1.arcsec' will not combine data for a field unless
their phase center is less than 1 arcsec.
respectname -- If true, fields with a different name are not merged even if their
direction agrees (within dirtol)
default: True
visweightscale -- list of the weight scales to be applied to the individual MSs
default: [] (don't modify weights, equivalent to setting scale to 1 for each MS)
keepcopy -- If true, a copy of the input MSs is kept in their original place.
default: false
copypointing -- If true, the POINTING table information will be present in the output.
If false, the result is an empty POINTING table.
default: True
"""
###
#Python script
tempdir = ''
originalvis = vislist
try:
casalog.origin('virtualconcat')
t = tbtool()
m = mstool()
#break the reference between vis and vislist as we modify vis
if (type(vislist) == str):
vis = [vislist]
else:
vis = list(vislist)
#dto. for concavis
theconcatvis = concatvis
doweightscale = False
if (len(visweightscale) > 0):
if (len(visweightscale) != len(vis)):
raise Exception, 'parameter visweightscale must have same number of elements as parameter vis'
for factor in visweightscale:
if factor < 0.:
raise Exception, 'parameter visweightscale must only contain positive numbers'
elif factor != 1.:
doweightscale = True
if ((type(concatvis) != str) or (len(concatvis.split()) < 1)):
raise Exception, 'Parameter concatvis is invalid.'
if (vis.count(concatvis) > 0):
raise Exception, 'Parameter concatvis must not be equal to one of the members of parameter vis.'
if (os.path.exists(concatvis)):
raise Exception, 'The output MMS must not yet exist.'
# process the input MSs in chronological order
sortedvis = []
sortedvisweightscale = []
namestuples = []
for name in vis:
t.open(name)
times = t.getcol('TIME')
t.close()
times.sort()
if doweightscale:
namestuples.append(
(times[0], name, visweightscale[vis.index(name)]))
else:
namestuples.append((times[0], name, 0))
sorted_namestuples = sorted(namestuples, key=lambda msname: msname[0])
for i in range(0, len(vis)):
sortedvis.append(sorted_namestuples[i][1])
sortedvisweightscale.append(sorted_namestuples[i][2])
if not vis == sortedvis:
casalog.post(
'The list of input MSs is not in chronological order and will need to be sorted.',
'INFO')
casalog.post(
'The chronological order in which the concatenation will take place is:',
'INFO')
for name in sortedvis:
casalog.post(
' MJD ' + str(
qa.splitdate(
qa.quantity(
sorted_namestuples[sortedvis.index(name)][0],
's'))['mjd']) + ': ' + name, 'INFO')
if doweightscale:
casalog.post(
'In this new order, the weights are:' +
str(sortedvisweightscale), 'INFO')
# replace the original vis and visweightscale by the sorted ones (with concatvis removed if it exists)
vis = sortedvis
visweightscale = sortedvisweightscale
# if there are MMSs among the input, make their constituents the new input
mmslist = []
ismaster = []
for elvis in vis:
ismaster.append(True) # may be revised later
if (ParallelTaskHelper.isParallelMS(elvis)):
mmslist.append(elvis)
if len(mmslist) > 0:
casalog.post(
'*** The following input measurement sets are multi-MSs',
'INFO')
for mname in mmslist:
casalog.post('*** ' + mname, 'INFO')
oldvis = vis
oldvisweightscale = visweightscale
vis = []
visweightscale = []
ismaster = [] # reset ismaster
i = 0
for elvis in oldvis:
if elvis in mmslist: # append the subMSs individually
m.open(elvis)
mses = m.getreferencedtables()
m.close()
mses.sort()
mastername = os.path.basename(
os.path.dirname(os.path.realpath(elvis + '/ANTENNA')))
for mname in mses:
#print 'subms: ', mname
vis.append(mname)
if doweightscale:
visweightscale.append(oldvisweightscale[i])
if os.path.basename(mname) == mastername:
ismaster.append(True)
else:
ismaster.append(False)
else:
vis.append(elvis)
if doweightscale:
visweightscale.append(oldvisweightscale[i])
ismaster.append(True)
i += 1
if keepcopy:
casalog.post(
'*** keepcopy==True: creating copy of input MSs to keep ...',
'INFO')
tempdir = 'concat_tmp_' + str(time.time())
os.mkdir(tempdir)
for elvis in originalvis:
shutil.move(elvis, tempdir) # keep timestamps and permissions
shutil.copytree(tempdir + '/' + elvis, elvis,
True) # symlinks=True
casalog.post('Concatenating ...', 'INFO')
if not copypointing: # delete the rows of all pointing tables
casalog.post(
'*** copypointing==False: resulting MMS will have empty POINTING table.',
'INFO')
tmptabname = 'TMPPOINTING' + str(time.time())
tmptabname2 = 'TMPPOINTING2' + str(time.time())
shutil.rmtree(tmptabname, ignore_errors=True)
shutil.rmtree(tmptabname2, ignore_errors=True)
shutil.move(vis[0] + '/POINTING', tmptabname)
t.open(tmptabname)
if (t.nrows() > 0):
ttab = t.copy(newtablename=tmptabname2,
deep=False,
valuecopy=True,
norows=True)
ttab.close()
t.close()
shutil.rmtree(tmptabname, ignore_errors=True)
else: # the POINTING table is already empty
t.close()
casalog.post('*** Input POINTING table was already empty.',
'INFO')
shutil.move(tmptabname, tmptabname2)
for i in range(
len(vis)): # replace the POINTING tables by the empty one
os.system('rm -rf ' + vis[i] + '/POINTING')
shutil.copytree(tmptabname2, vis[i] + '/POINTING')
shutil.rmtree(tmptabname2, ignore_errors=True)
if (len(vis) >
0): # (note: in case len is 1, we only copy, essentially)
theconcatvis = vis[0]
if (len(vis) == 1):
shutil.copytree(vis[0], concatvis, True)
vis.remove(vis[0])
# Determine if scratch columns should be considered at all
# by checking if any of the MSs has them.
considerscrcols = False
needscrcols = []
if ((type(theconcatvis) == str) and (os.path.exists(theconcatvis))):
# check if all scratch columns are present
t.open(theconcatvis)
if (t.colnames().count('CORRECTED_DATA') == 1
or t.colnames().count('MODEL_DATA') == 1):
considerscrcols = True # there are scratch columns
needscrcols.append(t.colnames().count('CORRECTED_DATA') == 0
or t.colnames().count('MODEL_DATA') == 0)
t.close()
else:
raise Exception, 'Visibility data set ' + theconcatvis + ' not found - please verify the name'
for elvis in vis: ###Oh no Elvis does not exist Mr Bill
if (not os.path.exists(elvis)):
raise Exception, 'Visibility data set ' + elvis + ' not found - please verify the name'
# check if all scratch columns are present
t.open(elvis)
if (t.colnames().count('CORRECTED_DATA') == 1
or t.colnames().count('MODEL_DATA') == 1):
considerscrcols = True # there are scratch columns
needscrcols.append(t.colnames().count('CORRECTED_DATA') == 0
or t.colnames().count('MODEL_DATA') == 0)
t.close()
# start actual work, file existence has already been checked
if (considerscrcols and needscrcols[0]):
# create scratch cols
casalog.post('creating scratch columns in ' + theconcatvis, 'INFO')
_cb.open(theconcatvis) # calibrator-open creates scratch columns
_cb.close()
# scale the weights of the first MS in the chain
if doweightscale:
wscale = visweightscale[0]
if (wscale == 1.):
casalog.post('Will leave the weights for this MS unchanged.',
'INFO')
else:
casalog.post(
'Scaling weights for first MS by factor ' + str(wscale),
'INFO')
t.open(theconcatvis, nomodify=False)
for colname in ['WEIGHT', 'WEIGHT_SPECTRUM']:
if (colname in t.colnames()) and (t.iscelldefined(
colname, 0)):
for j in xrange(0, t.nrows()):
a = t.getcell(colname, j)
a *= wscale
t.putcell(colname, j, a)
t.close()
m.open(theconcatvis, nomodify=False)
mmsmembers = [theconcatvis]
auxfile = 'concat_aux_' + str(time.time())
i = 0
for elvis in vis:
i = i + 1
mmsmembers.append(elvis)
casalog.post('adding ' + elvis + ' to multi-MS ' + concatvis,
'INFO')
wscale = 1.
if doweightscale:
wscale = visweightscale[i]
if (wscale == 1.):
casalog.post(
'Will leave the weights for this MS unchanged.',
'INFO')
else:
casalog.post(
'Will scale weights for this MS by factor ' +
str(wscale), 'INFO')
if (considerscrcols and needscrcols[i]):
# create scratch cols
casalog.post('creating scratch columns for ' + elvis, 'INFO')
_cb.open(elvis) # calibrator-open creates scratch columns
_cb.close()
m.virtconcatenate(msfile=elvis,
auxfilename=auxfile,
freqtol=freqtol,
dirtol=dirtol,
respectname=respectname,
weightscale=wscale)
#end for
os.remove(auxfile)
m.close()
# Write history to MS
try:
param_names = virtualconcat.func_code.co_varnames[:virtualconcat.
func_code.
co_argcount]
param_vals = [eval(p) for p in param_names]
write_history(mstool(), theconcatvis, 'virtualconcat', param_names,
param_vals, casalog)
except Exception, instance:
casalog.post("*** Error \'%s\' updating HISTORY" % (instance),
'WARN')
# concatenate the POINTING tables
masterptable = mmsmembers[0] + '/POINTING'
ptablemembers = []
if os.path.exists(masterptable) and copypointing:
casalog.post('Concatenating the POINTING tables ...', 'INFO')
i = 0
for i in xrange(len(mmsmembers)):
ptable = mmsmembers[i] + '/POINTING'
if ismaster[i] and os.path.exists(ptable):
casalog.post(' ' + ptable, 'INFO')
shutil.move(ptable, ptable + str(i))
ptablemembers.append(ptable + str(i))
#end for
t.createmultitable(masterptable, ptablemembers, 'SUBTBS')
# endif
# Get all available subtables
thesubtables = ph.getSubtables(mmsmembers[0])
# Remove the SOURCE and HISTORY tables, which will be the only copied.
# All other sub-tables will be linked to first subms
thesubtables.remove('SOURCE')
thesubtables.remove('HISTORY')
subtabs_to_omit = thesubtables
ph.makeMMS(
concatvis,
mmsmembers,
True, # copy subtables from first to all other members
subtabs_to_omit) # excluding tables which will be linked
# remove the remaining "hulls" of the emptied input MMSs (if there are any)
for elvis in mmslist:
shutil.rmtree(elvis)
if keepcopy:
for elvis in originalvis:
shutil.move(tempdir + '/' + elvis, elvis)
os.rmdir(tempdir)
