def VLAUVLoadT(filename, disk, Aname, Aclass, Adisk, Aseq, err, \
Compress=False):
""" Read FITS file into AIPS
Read input uvtab FITS file, write AIPS
Returns Obit uv object
Filename = input FITS uvtab format file
disk = input FITS file disk number
Aname = output AIPS file name
Aclass = output AIPS file class
Adisk = output AIPS file disk
Aseq = output AIPS file sequence
err = Obit error/message stack
Compress = Write AIPS data in compressed form?
"""
################################################################
#
uvc = ObitTask.ObitTask("UVCopy")
uvc.DataType = "FITS"
uvc.inFile = filename
uvc.inDisk = disk
uvc.outDType = "AIPS"
uvc.outName = Aname
uvc.outClass = Aclass
uvc.outSeq = Aseq
uvc.outDisk = Adisk
uvc.Compress = Compress
uvc.g
# Get output
outuv = UV.newPAUV("UVdata", Aname, Aclass, Adisk, Aseq, True, err)
return outuv
def IDIFix(uv, err):
"""
Fix VLBA data loaded from IDI files.
* uv = UV data set to be sorted
* err = OErr object
Returns the corrected UV data object.
"""
# Sort data
uvsrt = AIPSTask.AIPSTask('uvsrt')
OTObit.setname(uv, uvsrt)
uvsrt.outname = uvsrt.inname
uvsrt.outclass = uvsrt.inclass
uvsrt.outseq = uvsrt.inseq + 1
uvsrt.outdisk = uvsrt.indisk
uvsrt.sort = 'TB'
uvsrt.go()
# Get output UV data object
uvFixed = UV.newPAUV("uvFixed", uvsrt.outname, uvsrt.outclass,
int(uvsrt.outdisk), int(uvsrt.outseq), True, err)
# Fix tables
MergeCal.MergeCal(uvFixed, err)
# Create NX table
indxr = AIPSTask.AIPSTask('indxr')
OTObit.setname(uvFixed, indxr)
indxr.go()
return uvFixed
def getname(cno, disk=Adisk):
""" Return Obit object for AIPS file in cno on disk
cno = AIPS catalog slot number
disk = AIPS disk number
"""
################################################################
Adisk = disk
user = AIPS.AIPS.userno
s = AIPSDir.PInfo(disk, user, cno, err)
OErr.printErrMsg(err, "Error with AIPS catalog")
# parse returned string
Aname = s[0:12]
Aclass = s[13:19]
Aseq = int(s[20:25])
Atype = s[26:28]
if Atype == 'MA':
out = Image.newPAImage("AIPS image", Aname, Aclass, disk, Aseq, True, err)
print "AIPS Image",Aname, Aclass, disk, Aseq
elif Atype == 'UV':
out = UV.newPAUV("AIPS UV data", Aname, Aclass, disk, Aseq, True, err)
print "AIPS UV",Aname, Aclass, disk, Aseq
out.Aname = Aname
out.Aclass = Aclass
out.Aseq = Aseq
out.Atype = Atype
out.Disk = disk
out.Acno = cno
return out
def __init__(self, name, klass, disk, seq):
self._err = OErr.OErr()
OSystem.PSetAIPSuser(AIPS.userno)
self._data = UV.newPAUV(name, name, klass, disk, seq, True, self._err)
if self._err.isErr:
raise RuntimeError
return
def getname(cno, disk=Adisk):
"""
Return Obit object for AIPS file in cno on disk
* cno = AIPS catalog slot number
* disk = AIPS disk number
"""
################################################################
Adisk = disk
user = AIPS.AIPS.userno
s = AIPSDir.PInfo(disk, user, cno, err)
OErr.printErrMsg(err, "Error with AIPS catalog")
# parse returned string
Aname = s[0:12]
Aclass = s[13:19]
Aseq = int(s[20:25])
Atype = s[26:28]
if Atype == 'MA':
out = Image.newPAImage("AIPS image", Aname, Aclass, disk, Aseq, True,
err)
print("AIPS Image", Aname, Aclass, disk, Aseq)
elif Atype == 'UV':
out = UV.newPAUV("AIPS UV data", Aname, Aclass, disk, Aseq, True, err)
print("AIPS UV", Aname, Aclass, disk, Aseq)
out.Aname = Aname
out.Aclass = Aclass
out.Aseq = Aseq
out.Atype = Atype
out.Disk = disk
out.Acno = cno
return out
def VLAUVLoadT(filename, disk, Aname, Aclass, Adisk, Aseq, err, \
Compress=False):
""" Read FITS file into AIPS
Read input uvtab FITS file, write AIPS
Returns Obit uv object
Filename = input FITS uvtab format file
disk = input FITS file disk number
Aname = output AIPS file name
Aclass = output AIPS file class
Adisk = output AIPS file disk
Aseq = output AIPS file sequence
err = Obit error/message stack
Compress = Write AIPS data in compressed form?
"""
################################################################
#
uvc = ObitTask.ObitTask("UVCopy")
uvc.DataType = "FITS"
uvc.inFile = filename
uvc.inDisk = disk
uvc.outDType = "AIPS"
uvc.outName = Aname
uvc.outClass = Aclass
uvc.outSeq = Aseq
uvc.outDisk = Adisk
uvc.Compress = Compress
uvc.g
# Get output
outuv = UV.newPAUV("UVdata", Aname, Aclass, Adisk, Aseq, True, err)
return outuv
def PAllDest(disk, err, Atype=" ", Aname=None, Aclass=None, Aseq=0):
"""
Delete selected AIPS catalog entries
Entries can be selected using Atype, Aname, Aclass, Aseq,
using AIPS wildcards
A "?" matches one of any character, "*" matches any string
all blanks matches anything
* disk = AIPS disk number, 0=>all
* err = Python Obit Error/message stack
* type = optional file type
* Aname = desired name, using AIPS wildcards, None -> don't check
* Aclass = desired class, using AIPS wildcards, None -> don't check
* Aseq = desired sequence, 0=> any
* first = optional first slot number (1-rel)
* last = optional last slot number
* giveList = If true, return list of CNOs matching
"""
################################################################
# Checks
if not OErr.OErrIsA(err):
raise TypeError, "err MUST be an OErr"
#
if (disk < 0) or (disk > (len(AIPS.AIPS.disks) + 1)):
raise RuntimeError, "Disk " + str(disk) + " out of range"
# disks
if disk > 0:
disks = [disk]
else:
disks = range(1, len(AIPS.AIPS.disks))
user = OSystem.PGetAIPSuser()
# loop over disks
for dsk in disks:
ncno = PNumber(dsk, user, err)
OErr.printErrMsg(err, "Error getting number of cnos")
for cno in range(1, ncno):
line = PInfo(dsk, user, cno, err)
OErr.printErrMsg(err, "Error reading entry")
if WantDir(line, type=Atype, Aname=Aname, Aclass=Aclass,
Aseq=Aseq):
# parse directory string
Tname = line[0:12]
Tclass = line[13:19]
Tseq = int(line[20:25])
Ttype = line[26:28]
z = None
if Ttype == 'MA':
z = Image.newPAImage("Zap image", Tname, Tclass, dsk, Tseq, True, err, \
verbose=False)
print "Zap AIPS Image", Tname, Tclass, dsk, Tseq
elif Ttype == 'UV':
z = UV.newPAUV("Zap UV data", Tname, Tclass, dsk, Tseq, True, err, \
verbose=False)
print "Zap AIPS UV", Tname, Tclass, dsk, Tseq
# Delete entry
if z:
z.Zap(err)
del z
def PAllDest(disk, err, Atype = " ", Aname=None, Aclass=None, Aseq=0):
""" Delete selected AIPS catalog entries
Entries can be selected using Atype, Aname, Aclass, Aseq,
using AIPS wildcards
A "?" matches one of any character, "*" matches any string
all blanks matches anything
disk = AIPS disk number, 0=>all
err = Python Obit Error/message stack
type = optional file type
Aname = desired name, using AIPS wildcards, None -> don't check
Aclass = desired class, using AIPS wildcards, None -> don't check
Aseq = desired sequence, 0=> any
first = optional first slot number (1-rel)
last = optional last slot number
giveList = If true, return list of CNOs matching
"""
################################################################
# Checks
if not OErr.OErrIsA(err):
raise TypeError,"err MUST be an OErr"
#
if (disk<0) or (disk>(len(AIPS.AIPS.disks)+1)):
raise RuntimeError,"Disk "+str(disk)+" out of range"
# disks
if disk>0:
disks=[disk]
else:
disks= range(1,len(AIPS.AIPS.disks))
user = OSystem.PGetAIPSuser()
# loop over disks
for dsk in disks:
ncno = PNumber (dsk, user, err)
OErr.printErrMsg(err, "Error getting number of cnos")
for cno in range(1, ncno):
line=PInfo(dsk, user, cno, err);
OErr.printErrMsg(err, "Error reading entry")
if WantDir(line, type=Atype, Aname=Aname, Aclass=Aclass,
Aseq=Aseq):
# parse directory string
Tname = line[0:12]
Tclass = line[13:19]
Tseq = int(line[20:25])
Ttype = line[26:28]
z = None
if Ttype == 'MA':
z = Image.newPAImage("Zap image", Tname, Tclass, dsk, Tseq, True, err, \
verbose=False)
print "Zap AIPS Image",Tname, Tclass, dsk, Tseq
elif Ttype == 'UV':
z = UV.newPAUV("Zap UV data", Tname, Tclass, dsk, Tseq, True, err, \
verbose=False)
print "Zap AIPS UV",Tname, Tclass, dsk, Tseq
# Delete entry
if z:
z.Zap(err)
del z
def VLAUVLoad(filename, inDisk, Aname, Aclass, Adisk, Aseq, err, logfile=''):
""" Read FITS uvtab file into AIPS
Read a UVTAB FITS UV data file and write an AIPS data set
filename = name of FITS file
inDisk = FITS directory number
Aname = AIPS name of file
Aclass = AIPS class of file
Aseq = AIPS sequence number of file, 0=> create new
Adisk = FITS directory number
err = Python Obit Error/message stack
logfile = logfile for messages
returns AIPS UV data object
"""
################################################################
#
# Checks
if not OErr.OErrIsA(err):
raise TypeError,"err MUST be an OErr"
#
# Get input
inUV = UV.newPFUV("FITS UV DATA", filename, inDisk, True, err)
if err.isErr:
OErr.printErrMsg(err, "Error with FITS data")
# Get output, create new if seq=0
if Aseq<1:
OErr.printErr(err) # Print any outstanding messages
user = OSystem.PGetAIPSuser()
Aseq=AIPSDir.PHiSeq(Adisk,user,Aname,Aclass,"MA",err)
# If it already exists, increment seq
if AIPSDir.PTestCNO(Adisk,user,Aname,Aclass,"MA",Aseq,err)>0:
Aseq = Aseq+1
OErr.PClear(err) # Clear any message/error
mess = "Creating AIPS UV file "+Aname+"."+Aclass+"."+str(Aseq)+" on disk "+str(Adisk)
printMess(mess, logfile)
outUV = UV.newPAUV("AIPS UV DATA", Aname, Aclass, Adisk, Aseq, False, err)
if err.isErr:
OErr.printErrMsg(err, "Error creating AIPS data")
# Copy
UV.PCopy (inUV, outUV, err)
if err.isErr:
OErr.printErrMsg(err, "Error copying UV data to AIPS")
# Copy History
inHistory = History.History("inhistory", inUV.List, err)
outHistory = History.History("outhistory", outUV.List, err)
History.PCopyHeader(inHistory, outHistory, err)
# Add history
outHistory.Open(History.READWRITE, err)
outHistory.TimeStamp(" Start Obit uvlod",err)
outHistory.WriteRec(-1,"uvlod / FITS file "+filename+" disk "+str(inDisk),err)
outHistory.Close(err)
#
# Copy Tables
exclude=["AIPS HI", "AIPS AN", "AIPS FQ", "AIPS SL", "AIPS PL", "History"]
include=[]
UV.PCopyTables (inUV, outUV, exclude, include, err)
return outUV # return new object
def _init(self, desc, verbose=True):
userno = OSystem.PGetAIPSuser()
OSystem.PSetAIPSuser(desc['userno'])
uvdata = UV.newPAUV(desc['name'], desc['name'], desc['klass'],
desc['disk'], desc['seq'], True, self.err,
verbose = verbose)
OSystem.PSetAIPSuser(userno)
if not uvdata.isOK: # Exception if something went wrong
raise OErr.OErr
OErr.printErrMsg(self.err, "Error with AIPSUVdata")
return uvdata
def pipeline(scriptName, aipsSetup, parmFile):
"""
Linear (feed) Polarization Continuum pipeline.
* *scriptName* = this script file name
* *aipsSetup* = AIPS setup file
* *parmFile* = pipeline input parameters file
"""
############################# Initialize OBIT ##########################################
noScrat = []
exec(open(aipsSetup).read())
EVLAAddOutFile(aipsSetup, 'project', "Obit's AIPS setup file")
############################# Default parameters ##########################################
# Initialize parameters
parms = EVLAInitContParms()
# Linear feed specific parameters
# Relative gain calibration
parms["doXYRelGain"] = False # constrain relative X, Y gain amplitudes
parms[
"doXYRelGain2"] = None # constrain relative X, Y gain amplitudes, 2nd pass
parms["XYRelGainCal"] = None # X/Y gain calibrator structure
parms["XYRelGainCalTime"] = [0., 0.
] # Time range for relative gain calibration
############################# Set Project Processing parameters ##################
print("parmFile", parmFile)
exec(open(parmFile).read())
EVLAAddOutFile(parmFile, 'project', 'Pipeline input parameters')
# frequency/configuration dependent default parameters
EVLAInitContFQParms(parms)
# Linear feed specific:
parms["avgPol"] = parms["doXYRelGain"] and (parms["XYRelGainCal"] != None
) # Fix relative X/Y gains?
# General data parameters
band = parms["band"] # Observing band
dataClass = ("UVDa" + band)[0:6] # AIPS class of raw uv data
project = parms["project"][
0:12] # Project name (12 char or less, used as AIPS Name)
session = parms["session"] # Project session code
################################## Process #####################################
fileRoot = parms["project"] + "_" + parms["session"] + "_" + parms[
"band"] # root of file name
logFile = fileRoot + ".log" # Processing log file
uv = None
uvc = None
avgClass = ("UVAv" + band)[0:6] # Averaged data AIPS class
outIClass = parms["outIClass"] # image AIPS class
# Load the outputs pickle jar
EVLAFetchOutFiles()
# Logging directly to logFile
OErr.PInit(err, parms["prtLv"], logFile)
OSystem.PAllowThreads(nThreads) # Allow threads in Obit/oython
retCode = 0
EVLAAddOutFile(logFile, 'project', 'Pipeline log file')
mess = "Start project "+parms["project"]+" session "+parms["session"]+\
" "+parms["band"]+" Band"+" AIPS user no. "+str(AIPS.userno)+\
", EVLA configuration "+parms["VLACfg"]
printMess(mess, logFile)
if debug:
pydoc.ttypager = pydoc.plainpager # don't page task input displays
mess = "Using Debug mode "
printMess(mess, logFile)
if check:
mess = "Only checking script"
printMess(mess, logFile)
# Log parameters
printMess("Parameter settings", logFile)
for p in parms:
mess = " " + p + ": " + str(parms[p])
printMess(mess, logFile)
# Save parameters to pickle jar, manifest
ParmsPicklefile = project + "_" + session + "_" + band + ".Parms.pickle" # Where results saved
SaveObject(parms, ParmsPicklefile, True)
EVLAAddOutFile(ParmsPicklefile, 'project', 'Processing parameters used')
# Are we going to be doing Hanning?
if parms["doHann"]:
loadClass = parms["band"] + "Raw"
else:
loadClass = dataClass
# Load Data from Archive directory
if parms["doLoadArchive"]:
uv = EVLAUVLoadArch(parms["archRoot"], EVLAAIPSName(project, session), loadClass, disk, parms["seq"], err, \
selConfig=parms["selConfig"], doSwPwr=parms["doSwPwr"], \
selBand=parms["selBand"], selChan=parms["selChan"], \
selNIF=parms["selNIF"], calInt=parms["calInt"], \
logfile=logFile, Compress=parms["Compress"], check=check, debug=debug)
if uv == None and not check:
raise RuntimeError("Cannot load " + parms["DataRoot"])
# Hanning
if parms["doHann"]:
# Set uv if not done
if uv == None and not check:
uv = UV.newPAUV("AIPS UV DATA", EVLAAIPSName(project, session),
loadClass[0:6], disk, parms["seq"], True, err)
if err.isErr:
OErr.printErrMsg(err, "Error creating AIPS data")
uv = EVLAHann(uv, EVLAAIPSName(project, session), dataClass, disk, parms["seq"], err, \
doDescm=parms["doDescm"], logfile=logFile, check=check, debug=debug)
if uv == None and not check:
raise RuntimeError("Cannot Hann data ")
# Set uv if not done
if uv == None and not check:
uv = UV.newPAUV("AIPS UV DATA", EVLAAIPSName(project, session), dataClass[0:6], \
disk, parms["seq"], True, err)
if err.isErr:
OErr.printErrMsg(err, "Error creating AIPS data")
# Save file names in history
EVLAScriptHistory(uv, scriptName, aipsSetup, parmFile, err)
if err.isErr:
OErr.printErrMsg(err, "Error writing file names to history")
# Clear any old calibration/editing
if parms["doClearTab"]:
mess = "Clear previous calibration"
printMess(mess, logFile)
EVLAClearCal(uv,
err,
doGain=parms["doClearGain"],
doFlag=parms["doClearFlag"],
doBP=parms["doClearBP"],
check=check)
OErr.printErrMsg(err, "Error resetting calibration")
# Copy FG 1 to FG 2
if parms["doCopyFG"]:
mess = "Copy FG 1 to FG 2"
printMess(mess, logFile)
retCode = EVLACopyFG(uv,
err,
logfile=logFile,
check=check,
debug=debug)
if retCode != 0:
raise RuntimeError("Error Copying FG table")
# Drop end channels of spectra? Only if new FG 2
if parms["doCopyFG"] and (parms["BChDrop"] > 0) or (parms["EChDrop"] > 0):
# Channels based on original number, reduced if Hanning
nchan = uv.Desc.Dict["inaxes"][uv.Desc.Dict["jlocf"]]
fact = parms["selChan"] / nchan # Hanning reduction factor
BChDrop = parms["BChDrop"] / fact
EChDrop = parms["EChDrop"] / fact
mess = "Trim %d channels from start and %d from end of each spectrum" % (
BChDrop, EChDrop)
printMess(mess, logFile)
retCode = EVLADropChan (uv, BChDrop, EChDrop, err, flagVer=parms["editFG"], \
logfile=logFile, check=check, debug=debug)
if retCode != 0:
raise RuntimeError("Error Copying FG table")
# Special editing
if parms["doEditList"] and not check:
mess = "Special editing"
printMess(mess, logFile)
for edt in parms["editList"]:
UV.PFlag(uv,err,timeRange=[dhms2day(edt["timer"][0]),dhms2day(edt["timer"][1])], \
flagVer=parms["editFG"], Ants=edt["Ant"], Chans=edt["Chans"], IFs=edt["IFs"], \
Stokes=edt["Stokes"], Reason=edt["Reason"])
OErr.printErrMsg(err, "Error Flagging")
# Quack to remove data from start and end of each scan
if parms["doQuack"]:
retCode = EVLAQuack (uv, err, begDrop=parms["quackBegDrop"], endDrop=parms["quackEndDrop"], \
Reason=parms["quackReason"], \
logfile=logFile, check=check, debug=debug)
if retCode != 0:
raise RuntimeError("Error Quacking data")
# Flag antennas shadowed by others?
if parms["doShad"]:
retCode = EVLAShadow (uv, err, shadBl=parms["shadBl"], \
logfile=logFile, check=check, debug=debug)
if retCode != 0:
raise RuntimeError("Error Shadow flagging data")
# Median window time editing, for RFI impulsive in time
if parms["doMedn"]:
mess = "Median window time editing, for RFI impulsive in time:"
printMess(mess, logFile)
retCode = EVLAMedianFlag (uv, " ", err, noScrat=noScrat, nThreads=nThreads, \
avgTime=parms["avgTime"], avgFreq=parms["avgFreq"], chAvg= parms["chAvg"], \
timeWind=parms["timeWind"], flagVer=2,flagSig=parms["mednSigma"], \
logfile=logFile, check=check, debug=False)
if retCode != 0:
raise RuntimeError("Error in MednFlag")
# Median window frequency editing, for RFI impulsive in frequency
if parms["doFD1"]:
mess = "Median window frequency editing, for RFI impulsive in frequency:"
printMess(mess, logFile)
retCode = EVLAAutoFlag (uv, " ", err, flagVer=2, doCalib=-1, doBand=-1, \
timeAvg=parms["FD1TimeAvg"], \
doFD=True, FDmaxAmp=1.0e20, FDmaxV=1.0e20, FDwidMW=parms["FD1widMW"], \
FDmaxRMS=[1.0e20,0.1], FDmaxRes=parms["FD1maxRes"], \
FDmaxResBL= parms["FD1maxRes"], \
nThreads=nThreads, logfile=logFile, check=check, debug=debug)
if retCode != 0:
raise RuntimeError("Error in AutoFlag")
# RMS/Mean editing for calibrators
if parms["doRMSAvg"]:
mess = "RMS/Mean editing for calibrators:"
printMess(mess, logFile)
clist = [] # Calibrator list
for s in parms["ACals"]:
if s['Source'] not in clist:
clist.append(s['Source'])
for s in parms["PCals"]:
if s['Source'] not in clist:
clist.append(s['Source'])
for s in parms["DCals"]:
if s['Source'] not in clist:
clist.append(s['Source'])
retCode = EVLAAutoFlag (uv, clist, err, flagVer=2, doCalib=-1, doBand=-1, \
RMSAvg=parms["RMSAvg"], timeAvg=parms["RMSTimeAvg"], \
nThreads=nThreads, logfile=logFile, check=check, debug=debug)
if retCode != 0:
raise RuntimeError("Error in AutoFlag")
# Need to find a reference antenna? See if we have saved it?
if (parms["refAnt"] <= 0):
refAnt = FetchObject(project + "_" + session + "_" + band +
".refAnt.pickle")
if refAnt:
parms["refAnt"] = refAnt
# Use bandpass calibrator and center half of each spectrum
if parms["refAnt"] <= 0:
mess = "Find best reference antenna: run Calib on BP Cal(s) "
printMess(mess, logFile)
parms["refAnt"] = EVLAGetRefAnt(uv, parms["BPCals"], err, flagVer=2, \
solInt=parms["bpsolint1"], nThreads=nThreads, \
logfile=logFile, check=check, debug=debug)
if err.isErr:
raise RuntimeError("Error finding reference antenna")
if parms["refAnts"][0] <= 0:
parms["refAnts"][0] = parms["refAnt"]
mess = "Picked reference antenna " + str(parms["refAnt"])
printMess(mess, logFile)
# Save it
ParmsPicklefile = project + "_" + session + "_" + band + ".Parms.pickle" # Where results saved
SaveObject(parms, ParmsPicklefile, True)
refAntPicklefile = project + "_" + session + "_" + band + ".refAnt.pickle" # Where results saved
SaveObject(parms["refAnt"], refAntPicklefile, True)
# Plot Raw, edited data?
if parms["doRawSpecPlot"] and parms["plotSource"]:
mess = "Raw Spectral plot for: " + parms["plotSource"]
printMess(mess, logFile)
plotFile = "./" + fileRoot + "RawSpec.ps"
retCode = EVLASpectrum(uv, parms["plotSource"], parms["plotTime"], plotFile, parms["refAnt"], err, \
Stokes=["RR","LL"], doband=-1, \
check=check, debug=debug, logfile=logFile )
if retCode != 0:
raise RuntimeError("Error in Plotting spectrum")
EVLAAddOutFile(plotFile, 'project', 'Pipeline log file')
# delay calibration
if parms["doDelayCal"] and parms["DCals"] and not check:
plotFile = "./" + fileRoot + "DelayCal.ps"
retCode = EVLADelayCal(uv, parms["DCals"], err, \
BChan=parms["delayBChan"], EChan=parms["delayEChan"], \
doCalib=2, flagVer=2, doBand=-1, \
solInt=parms["delaySolInt"], smoTime=1.0/60.0, \
refAnts=[parms["refAnt"]], doTwo=parms["doTwo"],
doZeroPhs=parms["delayZeroPhs"], \
doPlot=parms["doSNPlot"], plotFile=plotFile, \
nThreads=nThreads, noScrat=noScrat, \
logfile=logFile, check=check, debug=debug)
if retCode != 0:
raise RuntimeError("Error in delay calibration")
# Plot corrected data?
if parms["doSpecPlot"] and parms["plotSource"]:
plotFile = "./" + fileRoot + "DelaySpec.ps"
retCode = EVLASpectrum(uv, parms["plotSource"], parms["plotTime"], \
plotFile, parms["refAnt"], err, \
Stokes=["RR","LL"], doband=-1, \
check=check, debug=debug, logfile=logFile )
if retCode != 0:
raise RuntimeError("Error in Plotting spectrum")
# Bandpass calibration
if parms["doBPCal"] and parms["BPCals"]:
retCode = EVLABPCal(uv, parms["BPCals"], err, noScrat=noScrat, solInt1=parms["bpsolint1"], \
solInt2=parms["bpsolint2"], solMode=parms["bpsolMode"], \
BChan1=parms["bpBChan1"], EChan1=parms["bpEChan1"], \
BChan2=parms["bpBChan2"], EChan2=parms["bpEChan2"], ChWid2=parms["bpChWid2"], \
doCenter1=parms["bpDoCenter1"], refAnt=parms["refAnt"], \
UVRange=parms["bpUVRange"], doCalib=2, gainUse=0, flagVer=2, doPlot=False, \
nThreads=nThreads, logfile=logFile, check=check, debug=debug)
if retCode != 0:
raise RuntimeError("Error in Bandpass calibration")
# Plot corrected data?
if parms["doSpecPlot"] and parms["plotSource"]:
plotFile = "./" + fileRoot + "BPSpec.ps"
retCode = EVLASpectrum(uv, parms["plotSource"], parms["plotTime"], plotFile, \
parms["refAnt"], err, Stokes=["RR","LL"], doband=1, \
check=check, debug=debug, logfile=logFile )
if retCode != 0:
raise RuntimeError("Error in Plotting spectrum")
# Constrain X/Y gain
if parms["doXYRelGain"] and (parms["XYRelGainCal"] != None):
mess = "Calibrate X/Y relative gain:"
printMess(mess, logFile)
plotFile = "./" + fileRoot + "XYGainCal.ps"
retCode = EVLACalAP (uv, [], parms["XYRelGainCal"], err,
doCalib=2, doBand=1, BPVer=1, flagVer=2, flagFail=False, \
BChan=parms["ampBChan"], EChan=parms["ampEChan"], \
solInt=parms["solInt"], solSmo=1440., ampScalar=parms["ampScalar"], \
doPlot=parms["doSNPlot"], plotFile=plotFile, refAnt=parms["refAnt"], \
nThreads=nThreads, noScrat=noScrat, logfile=logFile, check=check, debug=debug)
if retCode != 0:
raise RuntimeError("Error calibrating X/Y gain")
parms["avgPol"] = True
# Fix X/Y gain
# Amp & phase Calibrate
if parms["doAmpPhaseCal"]:
plotFile = "./" + fileRoot + "APCal.ps"
retCode = EVLACalAP (uv, [], parms["ACals"], err, PCals=parms["PCals"],
doCalib=2, doBand=1, BPVer=1, flagVer=2, avgPol=parms["avgPol"], \
BChan=parms["ampBChan"], EChan=parms["ampEChan"], \
solInt=parms["solInt"], solSmo=parms["solSmo"], ampScalar=parms["ampScalar"], \
doAmpEdit=parms["doAmpEdit"], ampSigma=parms["ampSigma"], \
ampEditFG=parms["ampEditFG"], \
doPlot=parms["doSNPlot"], plotFile=plotFile, refAnt=parms["refAnt"], \
nThreads=nThreads, noScrat=noScrat, logfile=logFile, check=check, debug=debug)
if retCode != 0:
raise RuntimeError("Error calibrating")
# More editing
if parms["doAutoFlag"]:
mess = "Post calibration editing:"
printMess(mess, logFile)
# if going to redo then only calibrators
if parms["doRecal"]:
# Only calibrators
clist = []
for DCal in DCals:
if DCal["Source"] not in clist:
clist.append(DCal["Source"])
for PCal in PCals:
if PCal["Source"] not in clist:
clist.append(DCal["Source"])
for ACal in ACals:
if ACal["Source"] not in clist:
clist.append(ACal["Source"])
else:
clist = []
retCode = EVLAAutoFlag (uv, clist, err, flagVer=2, \
doCalib=2, gainUse=0, doBand=1, BPVer=1, \
IClip=parms["IClip"], minAmp=parms["minAmp"], timeAvg=parms["timeAvg"], \
doFD=parms["doAFFD"], FDmaxAmp=parms["FDmaxAmp"], FDmaxV=parms["FDmaxV"], \
FDwidMW=parms["FDwidMW"], FDmaxRMS=parms["FDmaxRMS"], \
FDmaxRes=parms["FDmaxRes"], FDmaxResBL=parms["FDmaxResBL"], \
FDbaseSel=parms["FDbaseSel"], \
nThreads=nThreads, logfile=logFile, check=check, debug=debug)
if retCode != 0:
raise RuntimeError("Error in AutoFlag")
# Redo the calibration using new flagging?
if parms["doBPCal2"] == None:
parms["doBPCal2"] = parms["doBPCal"]
if parms["doDelayCal2"] == None:
parms["doDelayCal2"] = parms["doDelayCal2"]
if parms["doXYRelGain2"] == None:
parms["doXYRelGain2"] = parms["doXYRelGain"]
if parms["doAmpPhaseCal2"] == None:
parms["doAmpPhaseCal2"] = parms["doAmpPhaseCal"]
if parms["doAutoFlag2"] == None:
parms["doAutoFlagCal2"] = parms["doAutoFlag"]
if parms["doRecal"]:
mess = "Redo calibration:"
printMess(mess, logFile)
EVLAClearCal(uv,
err,
doGain=True,
doFlag=False,
doBP=True,
check=check,
logfile=logFile)
OErr.printErrMsg(err, "Error resetting calibration")
# Delay recalibration
if parms["doDelayCal2"] and parms["DCals"] and not check:
plotFile = "./" + fileRoot + "DelayCal2.ps"
retCode = EVLADelayCal(uv, parms["DCals"], err, \
BChan=parms["delayBChan"], EChan=parms["delayEChan"], \
doCalib=2, flagVer=2, doBand=-1, \
solInt=parms["delaySolInt"], smoTime=1.0/60.0, \
refAnts=[parms["refAnt"]], doTwo=parms["doTwo"], \
doZeroPhs=parms["delayZeroPhs"], \
doPlot=parms["doSNPlot"], plotFile=plotFile, \
nThreads=nThreads, noScrat=noScrat, \
logfile=logFile, check=check, debug=debug)
if retCode != 0:
raise RuntimeError("Error in delay calibration")
# Plot corrected data?
if parms["doSpecPlot"] and parms["plotSource"]:
plotFile = "./" + fileRoot + "DelaySpec2.ps"
retCode = EVLASpectrum(uv, parms["plotSource"], parms["plotTime"], plotFile, parms["refAnt"], err, \
Stokes=["RR","LL"], doband=-1, \
check=check, debug=debug, logfile=logFile )
if retCode != 0:
raise RuntimeError("Error in Plotting spectrum")
# Bandpass calibration
if parms["doBPCal2"] and parms["BPCals"]:
retCode = EVLABPCal(uv, parms["BPCals"], err, noScrat=noScrat, solInt1=parms["bpsolint1"], \
solInt2=parms["bpsolint2"], solMode=parms["bpsolMode"], \
BChan1=parms["bpBChan1"], EChan1=parms["bpEChan1"], \
BChan2=parms["bpBChan2"], EChan2=parms["bpEChan2"], ChWid2=parms["bpChWid2"], \
doCenter1=parms["bpDoCenter1"], refAnt=parms["refAnt"], \
UVRange=parms["bpUVRange"], doCalib=2, gainUse=0, flagVer=2, doPlot=False, \
nThreads=nThreads, logfile=logFile, check=check, debug=debug)
if retCode != 0:
raise RuntimeError("Error in Bandpass calibration")
# Plot corrected data?
if parms["doSpecPlot"] and parms["plotSource"]:
plotFile = "./" + fileRoot + "BPSpec2.ps"
retCode = EVLASpectrum(uv, parms["plotSource"], parms["plotTime"], plotFile, parms["refAnt"], err, \
Stokes=["RR","LL"], doband=1, \
check=check, debug=debug, logfile=logFile )
if retCode != 0:
raise RuntimeError("Error in Plotting spectrum")
# Constrain X/Y gain
parms["avgPol"] = False
# Fix X/Y gain?
if parms["doXYRelGain2"] and (parms["XYRelGainCal"] != None):
mess = "Calibrate X/Y relative gain:"
printMess(mess, logFile)
plotFile = "./" + fileRoot + "XYGainCal2.ps"
retCode = EVLACalAP (uv, [], parms["XYRelGainCal"], err,
doCalib=2, doBand=1, BPVer=1, flagVer=2, timeRange=parms["XYRelGainCalTime"], \
BChan=parms["ampBChan"], EChan=parms["ampEChan"], flagFail=False, \
solInt=parms["solInt"], solSmo=1440., ampScalar=parms["ampScalar"], \
doPlot=parms["doSNPlot"], plotFile=plotFile, refAnt=parms["refAnt"], \
nThreads=nThreads, noScrat=noScrat, logfile=logFile, check=check, debug=debug)
if retCode != 0:
raise RuntimeError("Error calibrating X/Y gain")
parms["avgPol"] = True
# Fix X/Y gain
# Amp & phase Recalibrate
if parms["doAmpPhaseCal2"]:
plotFile = "./" + fileRoot + "APCal2.ps"
retCode = EVLACalAP (uv, [], parms["ACals"], err, PCals=parms["PCals"], \
doCalib=2, doBand=1, BPVer=1, flagVer=2,avgPol=parms["avgPol"], \
BChan=parms["ampBChan"], EChan=parms["ampEChan"], \
solInt=parms["solInt"], solSmo=parms["solSmo"], ampScalar=parms["ampScalar"], \
doAmpEdit=parms["doAmpEdit"], ampSigma=parms["ampSigma"], \
ampEditFG=parms["ampEditFG"], \
doPlot=parms["doSNPlot"], plotFile=plotFile, refAnt=parms["refAnt"], \
noScrat=noScrat, nThreads=nThreads, logfile=logFile, check=check, debug=debug)
if retCode != 0:
raise RuntimeError("Error calibrating")
# More editing
if parms["doAutoFlag2"]:
mess = "Post recalibration editing:"
printMess(mess, logFile)
retCode = EVLAAutoFlag (uv, [], err, flagVer=2, \
doCalib=2, gainUse=0, doBand=1, BPVer=1, \
IClip=parms["IClip"], minAmp=parms["minAmp"], timeAvg=parms["timeAvg"], \
doFD=parms["doAFFD"], FDmaxAmp=parms["FDmaxAmp"], FDmaxV=parms["FDmaxV"], \
FDwidMW=parms["FDwidMW"], FDmaxRMS=parms["FDmaxRMS"], \
FDmaxRes=parms["FDmaxRes"], FDmaxResBL= parms["FDmaxResBL"], \
FDbaseSel=parms["FDbaseSel"], \
nThreads=nThreads, logfile=logFile, check=check, debug=debug)
if retCode != 0:
raise RuntimeError("Error in AutoFlag")
# end recal
# Calibrate and average data
if parms["doCalAvg"]:
retCode = EVLACalAvg (uv, avgClass, parms["seq"], parms["CalAvgTime"], err, \
flagVer=2, doCalib=2, gainUse=0, doBand=1, BPVer=1, doPol=False, \
avgFreq=parms["avgFreq"], chAvg=parms["chAvg"], \
BChan=parms["CABChan"], EChan=parms["CAEChan"], \
BIF=parms["CABIF"], EIF=parms["CAEIF"], Compress=parms["Compress"], \
nThreads=nThreads, logfile=logFile, check=check, debug=debug)
if retCode != 0:
raise RuntimeError("Error in CalAvg")
# Get calibrated/averaged data
if not check:
uv = UV.newPAUV("AIPS UV DATA", EVLAAIPSName(project, session), avgClass[0:6], \
disk, parms["seq"], True, err)
if err.isErr:
OErr.printErrMsg(err, "Error creating cal/avg AIPS data")
# XClip
if parms["XClip"] and parms["XClip"] > 0.0:
mess = "Cross Pol clipping:"
printMess(mess, logFile)
retCode = EVLAAutoFlag (uv, [], err, flagVer=-1, flagTab=1, \
doCalib=2, gainUse=0, doBand=-1, maxBad=1.0, \
XClip=parms["XClip"], timeAvg=1./60., \
nThreads=nThreads, logfile=logFile, check=check, debug=debug)
if retCode != 0:
raise RuntimeError("Error in AutoFlag")
# R-L delay calibration cal if needed,
if parms["doRLDelay"] and parms["RLDCal"][0][0] != None:
if parms["rlrefAnt"] <= 0:
parms["rlrefAnt"] = parms["refAnt"]
# parms["rlDoBand"] if before average, BPVer=parms["rlBPVer"],
retCode = EVLARLDelay(uv, err,\
RLDCal=parms["RLDCal"], BChan=parms["rlBChan"], \
EChan=parms["rlEChan"], UVRange=parms["rlUVRange"], \
soucode=parms["rlCalCode"], doCalib=parms["rlDoCal"], gainUse=parms["rlgainUse"], \
timerange=parms["rltimerange"], \
# NOT HERE doBand=parms["rlDoBand"], BPVer=parms["rlBPVer"], \
flagVer=parms["rlflagVer"], \
refAnt=parms["rlrefAnt"], doPol=False, \
nThreads=nThreads, noScrat=noScrat, logfile=logFile, \
check=check, debug=debug)
if retCode != 0:
raise RuntimeError("Error in R-L delay calibration")
# Polarization calibration
if parms["doPolCal"]:
if parms["PCRefAnt"] <= 0:
parms["PCRefAnt"] = parms["refAnt"]
retCode = EVLAPolCal(uv, parms["PCInsCals"], err, \
doCalib=2, gainUse=0, doBand=-1, flagVer=0, \
fixPoln=parms["PCFixPoln"], pmodel=parms["PCpmodel"], avgIF=parms["PCAvgIF"], \
solInt=parms["PCSolInt"], refAnt=parms["PCRefAnt"], solType=parms["PCSolType"], \
ChInc=parms["PCChInc"], ChWid=parms["PCChWid"], \
nThreads=nThreads, check=check, debug=debug, noScrat=noScrat, logfile=logFile)
if retCode != 0 and (not check):
raise RuntimeError("Error in polarization calibration: " +
str(retCode))
# end poln cal.
# R-L phase calibration cal., creates new BP table
if parms["doRLCal"] and parms["RLDCal"][0][0] != None:
plotFile = "./" + fileRoot + "RLSpec2.ps"
if parms["rlrefAnt"] <= 0:
parms["rlrefAnt"] = parms["refAnt"]
retCode = EVLARLCal(uv, err,\
RLDCal=parms["RLDCal"], BChan=parms["rlBChan"],
EChan=parms["rlEChan"], UVRange=parms["rlUVRange"], \
ChWid2=parms["rlChWid"], solInt1=parms["rlsolint1"], solInt2=parms["rlsolint2"], \
RLPCal=parms["RLPCal"], RLPhase=parms["RLPhase"], \
RM=parms["RLRM"], CleanRad=parms["rlCleanRad"], \
calcode=parms["rlCalCode"], doCalib=parms["rlDoCal"], gainUse=parms["rlgainUse"], \
timerange=parms["rltimerange"], FOV=parms["rlFOV"], \
doBand=-1, BPVer=1, flagVer=parms["rlflagVer"], \
refAnt=parms["rlrefAnt"], doPol=parms["doPol"], PDVer=parms["PDVer"], \
doPlot=parms["doSpecPlot"], plotFile=plotFile, \
nThreads=nThreads, noScrat=noScrat, logfile=logFile, \
check=check, debug=debug)
if retCode != 0:
raise RuntimeError("Error in RL phase spectrum calibration")
# VClip
if parms["VClip"] and parms["VClip"] > 0.0:
mess = "VPol clipping:"
printMess(mess, logFile)
retCode = EVLAAutoFlag (uv, [], err, flagVer=-1, flagTab=1, \
doCalib=2, gainUse=0, doBand=-1, \
VClip=parms["VClip"], timeAvg=parms["timeAvg"], \
nThreads=nThreads, logfile=logFile, check=check, debug=debug)
if retCode != 0:
raise RuntimeError("Error in AutoFlag VClip")
# Plot corrected data?
if parms["doSpecPlot"] and parms["plotSource"]:
plotFile = "./" + fileRoot + "Spec.ps"
retCode = EVLASpectrum(uv, parms["plotSource"], parms["plotTime"], \
plotFile, parms["refAnt"], err, \
Stokes=["RR","LL"], doband=-1, \
check=check, debug=debug, logfile=logFile )
if retCode != 0:
raise RuntimeError("Error in Plotting spectrum")
# Image targets
if parms["doImage"]:
# If targets not specified, image all
if len(parms["targets"]) <= 0:
slist = EVLAAllSource(uv,
err,
logfile=logFile,
check=check,
debug=debug)
else:
slist = parms["targets"]
EVLAImageTargets (uv, err, Sources=slist, seq=parms["seq"], sclass=outIClass, \
doCalib=2, doBand=1, flagVer=1, doPol=parms["doPol"], PDVer=parms["PDVer"], \
Stokes=parms["Stokes"], FOV=parms["FOV"], Robust=parms["Robust"], Niter=parms["Niter"], \
CleanRad=parms["CleanRad"], minFlux=parms["minFlux"], \
maxPSCLoop=parms["maxPSCLoop"], minFluxPSC=parms["minFluxPSC"], \
solPInt=parms["solPInt"], solPMode=parms["solPMode"], solPType=parms["solPType"], \
maxASCLoop=parms["maxASCLoop"], minFluxASC=parms["minFluxASC"], \
solAInt=parms["solAInt"], solAMode=parms["solAMode"], solAType=parms["solAType"], \
avgPol=parms["avgPol"], avgIF=parms["avgIF"], minSNR = 4.0, refAnt=parms["refAnt"], \
do3D=parms["do3D"], BLFact=parms["BLFact"], BLchAvg=parms["BLchAvg"], \
doMB=parms["doMB"], norder=parms["MBnorder"], maxFBW=parms["MBmaxFBW"], \
PBCor=parms["PBCor"],antSize=parms["antSize"], Beam=parms["Beam"], \
nTaper=parms["nTaper"], Tapers=parms["Tapers"], \
nThreads=nThreads, noScrat=noScrat, logfile=logFile, check=check, debug=debug)
# End image
# Get report on sources
if parms["doReport"]:
# If targets not specified, do all
if len(parms["targets"]) <= 0:
slist = EVLAAllSource(uv,
err,
logfile=logFile,
check=check,
debug=debug)
else:
slist = parms["targets"]
Report = EVLAReportTargets(uv, err, Sources=slist, seq=parms["seq"], sclass=outIClass, \
Stokes=parms["Stokes"], logfile=logFile, check=check, debug=debug)
# Save to pickle jar
ReportPicklefile = "./" + fileRoot + "Report.pickle" # Where results saved
SaveObject(Report, ReportPicklefile, True)
# Write results, cleanup
# Save cal/average UV data?
if parms["doSaveUV"] and (not check):
Aname = EVLAAIPSName(project, session)
cno = AIPSDir.PTestCNO(disk, user, Aname, avgClass[0:6], "UV",
parms["seq"], err)
if cno > 0:
uvt = UV.newPAUV("AIPS CAL UV DATA", Aname, avgClass, disk,
parms["seq"], True, err)
filename = parms["project"] + parms["session"] + parms[
"band"] + "Cal.uvtab"
fuv = EVLAUVFITS(uvt,
filename,
0,
err,
compress=parms["Compress"],
logfile=logFile)
EVLAAddOutFile(filename, 'project', "Calibrated Averaged UV data")
# Save list of output files
EVLASaveOutFiles()
del uvt
# Save raw UV data tables?
if parms["doSaveTab"] and (not check):
Aname = EVLAAIPSName(project, session)
cno = AIPSDir.PTestCNO(disk, user, Aname, dataClass[0:6], "UV",
parms["seq"], err)
if cno > 0:
uvt = UV.newPAUV("AIPS RAW UV DATA", Aname, dataClass[0:6], disk,
parms["seq"], True, err)
filename = parms["project"] + parms["session"] + parms[
"band"] + "CalTab.uvtab"
fuv = EVLAUVFITSTab(uvt, filename, 0, err, logfile=logFile)
EVLAAddOutFile(filename, 'project', "Calibrated AIPS tables")
del uvt
# Write History
filename = project + '_' + session + '_' + band + ".History.text"
OTObit.PrintHistory(uv, file=filename)
EVLAAddOutFile(filename, 'project',
"Processing history of calibrated data")
# Save list of output files
EVLASaveOutFiles()
# Imaging results
# If targets not specified, save all
if len(parms["targets"]) <= 0:
slist = EVLAAllSource(uv,
err,
logfile=logFile,
check=check,
debug=debug)
else:
slist = parms["targets"]
for target in slist:
if parms["doSaveImg"] and (not check):
for s in parms["Stokes"]:
oclass = s + outIClass[1:]
outname = target
# Test if image exists
cno = AIPSDir.PTestCNO(disk, user, outname, oclass, "MA",
parms["seq"], err)
if cno <= 0:
continue
x = Image.newPAImage("out", outname, oclass, disk,
parms["seq"], True, err)
outfile = "./" + fileRoot + target + "." + oclass + ".fits"
xf = EVLAImFITS(x, outfile, 0, err, logfile=logFile)
EVLAAddOutFile(outfile, target, 'Image of ' + target)
# Statistics
zz = imstat(x, err, logfile=logFile)
# end writing loop
# Save list of output files
EVLASaveOutFiles()
OErr.printErrMsg(err, "Writing output")
# Contour plots
if parms["doKntrPlots"]:
mess = "INFO --> Contour plots (doKntrPlots)"
printMess(mess, logFile)
EVLAKntrPlots(err,
imName=parms["targets"],
project=project,
session=session,
band=band,
disk=disk,
debug=debug)
# Save list of output files
EVLASaveOutFiles()
elif debug:
mess = "Not creating contour plots ( doKntrPlots = " + str(
parms["doKntrPlots"]) + " )"
printMess(mess, logFile)
# Source uv plane diagnostic plots
if parms["doDiagPlots"]:
mess = "INFO --> Diagnostic plots (doDiagPlots)"
printMess(mess, logFile)
# Get the highest number avgClass catalog file
Aname = EVLAAIPSName(project, session)
uvc = None
if not check:
uvname = project + "_" + session + "_" + band + "_Cal"
uvc = UV.newPAUV(uvname, Aname, avgClass, disk, parms["seq"], True,
err)
EVLADiagPlots( uvc, err, cleanUp=parms["doCleanup"], \
project=project, session=session, band=band, \
logfile=logFile, check=check, debug=debug )
# Save list of output files
EVLASaveOutFiles()
elif debug:
mess = "Not creating diagnostic plots ( doDiagPlots = " + str(
parms["doDiagPlots"]) + " )"
printMess(mess, logFile)
# Save metadata
srcMetadata = None
projMetadata = None
if parms["doMetadata"]:
mess = "INFO --> Save metadata (doMetadata)"
printMess(mess, logFile)
uvc = None
if not uvc:
# Get calibrated/averaged data
Aname = EVLAAIPSName(project, session)
uvname = project + "_" + session + "_" + band + "_Cal"
uvc = UV.newPAUV(uvname, Aname, avgClass, disk, parms["seq"], True,
err)
if err.isErr:
OErr.printErrMsg(err, "Error creating cal/avg AIPS data")
# Get source metadata; save to pickle file
srcMetadata = EVLASrcMetadata( uvc, err, Sources=parms["targets"], seq=parms["seq"], \
sclass=outIClass, Stokes=parms["Stokes"],\
logfile=logFile, check=check, debug=debug )
picklefile = "./" + fileRoot + ".SrcReport.pickle"
SaveObject(srcMetadata, picklefile, True)
EVLAAddOutFile(picklefile, 'project', 'All source metadata')
# Get project metadata; save to pickle file
projMetadata = EVLAProjMetadata( uvc, AIPS_VERSION, err, \
PCals=parms["PCals"], ACals=parms["ACals"], \
BPCals=parms["BPCals"], DCals=parms["DCals"], \
project = project, session = session, band = band, \
dataInUVF = parms["archRoot"], archFileID = 66666 )
picklefile = "./" + fileRoot + ".ProjReport.pickle"
SaveObject(projMetadata, picklefile, True)
EVLAAddOutFile(picklefile, 'project', 'Project metadata')
else:
# Fetch from pickle jar
picklefile = "./" + fileRoot + ".SrcReport.pickle"
srcMetadata = FetchObject(picklefile)
picklefile = "./" + fileRoot + ".ProjReport.pickle"
projMetadata = FetchObject(picklefile)
# Write report
if parms["doHTML"]:
mess = "INFO --> Write HTML report (doHTML)"
printMess(mess, logFile)
EVLAHTMLReport( projMetadata, srcMetadata, \
outfile="./"+fileRoot+".report.html", \
logFile=logFile )
# Write VOTable
if parms["doVOTable"]:
mess = "INFO --> Write VOTable (doVOTable)"
printMess(mess, logFile)
EVLAAddOutFile('VOTable.xml', 'project', 'VOTable report')
EVLAWriteVOTable(projMetadata, srcMetadata, filename='VOTable.xml')
# Save list of output files
EVLASaveOutFiles()
# Cleanup - delete AIPS files
if parms["doCleanup"] and (not check):
mess = "INFO --> Clean up (doCleanup)"
printMess(mess, logFile)
# Delete target images
# How many Stokes images
nstok = len(parms["Stokes"])
for istok in range(0, nstok):
oclass = parms["Stokes"][istok:istok + 1] + outIClass[1:]
AllDest(err, disk=disk, Aseq=parms["seq"], Aclass=oclass)
# Delete initial UV data
Aname = EVLAAIPSName(project, session)
# Test if data exists
cno = AIPSDir.PTestCNO(disk, user, Aname, dataClass[0:6], "UV",
parms["seq"], err)
if cno > 0:
uvt = UV.newPAUV("AIPS RAW UV DATA", Aname, dataClass[0:6], disk,
parms["seq"], True, err)
uvt.Zap(err)
del uvt
if err.isErr:
OErr.printErrMsg(err, "Error deleting raw AIPS data")
# Zap calibrated/averaged data
# Test if data exists
cno = AIPSDir.PTestCNO(disk, user, Aname, avgClass[0:6], "UV",
parms["seq"], err)
if cno > 0:
uvt = UV.newPAUV("AIPS CAL UV DATA", Aname, avgClass[0:6], disk,
parms["seq"], True, err)
uvt.Zap(err)
del uvt
if err.isErr:
OErr.printErrMsg(err, "Error deleting cal/avg AIPS data")
# Zap UnHanned data if present
loadClass = parms["band"] + "Raw"
# Test if image exists
cno = AIPSDir.PTestCNO(disk, user, Aname, loadClass[0:6], "UV",
parms["seq"], err)
if cno > 0:
uvt = UV.newPAUV("AIPS CAL UV DATA", Aname, loadClass[0:6], disk,
parms["seq"], True, err)
uvt.Zap(err)
del uvt
if err.isErr:
OErr.printErrMsg(err, "Error deleting cal/avg AIPS data")
OErr.printErrMsg(err, "Writing output/cleanup")
# Shutdown
mess = "Finished project "+parms["project"]+" session "+parms["session"]+ \
" "+parms["band"]+" Band"+" AIPS user no. "+str(AIPS.userno)
printMess(mess, logFile)
OErr.printErr(err)
OSystem.Shutdown(ObitSys)
def ImagePeel(uvsub, peelPos, err, \
nxy=512, Niter=1000, minFlux=0.001, \
maxPSCLoop=2, minFluxPSC=0.01, solPInt=1.0, \
minSNR = 3.5, Robust=0.0, doGPU=False, seq=1, \
nThreads=1, noScrat=[0,0,0], taskLog='', debug=False):
"""
Sets up to image subtracted uv data from UVSub4Peel, self calibrate peel source.
Only does A&P self cal
Returns MFImage task object, output image "IPlMod', uv 'UVPeel', Seq seq
* uvsub task object from UVSub4Peel
* peelPos [RA, Dec] in deg of source to peel
* err Python Obit Error/message stack
* nxy Size in pixels of x,y
* Niter max number of iterations
* minFlux Min flux density first CLEAN
* maxPSCLoop max number A&P self cal loops
* minFluxPSC min peak for self cal
* solPInt solution interval for self cal
* minSNR min SNR of self cal solutions
* Robust Briggs Robust factor
* seq Sequence number for output
* doGPU Use GPU if available?
* nThreads number of threads to use
* noScrat AIPS disks not to use for scratch
* taskLog Log file
* debug If True leave debug Input file in /tmp
"""
################################################################
# Checks
if not OErr.OErrIsA(err):
raise TypeError("err MUST be an OErr")
mfp = ObitTask('MFImage')
mfp.DataType = 'AIPS'
mfp.inName = uvsub.outName
mfp.inDisk = uvsub.outDisk
mfp.inSeq = uvsub.outSeq
mfp.inClass = uvsub.outClass
mfp.outDisk = mfp.inDisk
mfp.outSeq = seq
mfp.outClass = 'IPlMod'
mfp.out2Disk = mfp.inDisk
mfp.out2Seq = seq
mfp.out2Class = 'UVPeel'
# Shift
uv = UV.newPAUV('uv', mfp.inName, mfp.inClass, mfp.inDisk, mfp.inSeq, True,
err)
d = uv.Desc.Dict
ra = d['crval'][4]
dec = d['crval'][5]
shift = SkyGeom.PShiftXY(ra, dec, d['crota'][1], peelPos[0], peelPos[1])
mfp.RAShift[0] = 3600 * shift[0]
mfp.DecShift[0] = 3600 * shift[1]
mfp.nThreads = nThreads
mfp.noScrat = noScrat
mfp.Cmethod = 'DFT'
mfp.noNeg = False
mfp.doGPU = doGPU
mfp.PBCor = False
mfp.taskLog = taskLog
mfp.debug = debug
mfp.prtLv = 2
mfp.PBCor = False
mfp.NField = 1
mfp.nx[0] = nxy
mfp.ny[0] = nxy
mfp.OutlierDist = 0.0
mfp.minFlux = minFlux
mfp.Niter = Niter
mfp.Robust = Robust
mfp.minSNR = minSNR
mfp.maxPSCLoop = maxPSCLoop
mfp.minFluxPSC = minFluxPSC
mfp.solPInt = solPInt
mfp.solPType = 'L1'
mfp.solPMode = 'A&P'
mfp.maxASCLoop = 0
mfp.minFluxASC = minFluxPSC
mfp.solAInt = solPInt * 2
mfp.solAType = 'L1'
mfp.solAMode = 'A&P'
return mfp
if check:
mess = "Only checking script"
printMess(mess, logFile)
# Load Data from FITS
uv = None
if doLoadIDI:
uv = VLBAIDILoad(dataIn, project, session, band, dataClass, disk, seq, err, logfile=logFile, \
wtThresh=wtThresh, calInt=calInt, Compress=Compress, \
check=check, debug=debug)
if not UV.PIsA(uv):
raise RuntimeError, "Cannot load " + dataIn
# Otherwise set uv
if uv == None and not check:
Aname = VLBAAIPSName(project, session)
uv = UV.newPAUV("AIPS UV DATA", Aname, dataClass, disk, seq, True, err)
if err.isErr:
OErr.printErrMsg(err, "Error creating AIPS data")
# Clear any old calibration/editing
if doClearTab:
VLBAClearCal(uv,
err,
doGain=doGain,
doFlag=doFlag,
doBP=doBP,
check=check,
logfile=logFile)
OErr.printErrMsg(err, "Error resetting calibration")
# Copy FG 1 to FG 2
outSeq = 2666 # Single
outSeq = 1666 # Single
# NVSS catalog
Catalog = 'NVSSVZ.FIT'
OutlierDist = 1.0 # Maximum distance to add outlyers (deg)
OutlierFlux = 0.01 # Minimum estimated outlier flux density (Jy)
OutlierSI = -1.0 # Spectral index to estimate flux density
OutlierSize = 50 # Size of outlyer field
# Bombs away
#Bomb()
# Set data
print "Set data"
inData = UV.newPAUV("Input data", inName, inClass, inDisk, inSeq, True, err)
OErr.printErrMsg(err, "Error initializing")
# Make scratch version
print "Make scratch file"
scrData = UV.PScratch (inData, err)
OErr.printErrMsg(err, "Error making scratch file")
# Calibration/selection parameters
Input = UVImager.UVCalInput
Input['InData'] = inData
Input['OutData'] = scrData
Input['doCalSelect'] = True
Input['Stokes'] = 'I'
Input['BChan'] = 0
Input['EChan'] = 0
def pipeline(aipsSetup, parmFile):
"""
ALMA Continuum pipeline.
* *aipsSetup* = AIPS setup file
* *parmFile* = pipeline input parameters file
"""
############################# Initialize OBIT ##########################################
noScrat = []
exec(open(aipsSetup).read())
ALMAAddOutFile(aipsSetup, 'project', "Obit's AIPS setup file")
############################# Default parameters ##########################################
# Initialize parameters
parms = ALMAInitContParms()
############################# Set Project Processing parameters ##################
print "parmFile", parmFile
exec(open(parmFile).read())
ALMAAddOutFile(parmFile, 'project', 'Pipeline input parameters')
# frequency/configuration dependent default parameters
ALMAInitContFQParms(parms)
# General data parameters
band = parms["band"] # Observing band
dataClass = ("UVDa" + band)[0:6] # AIPS class of raw uv data
project = parms["project"][
0:12] # Project name (12 char or less, used as AIPS Name)
session = parms["session"] # Project session code
################################## Process #####################################
fileRoot = parms["project"] + "_" + parms["session"] + "_" + parms[
"band"] # root of file name
logFile = fileRoot + ".log" # Processing log file
uv = None
uvc = None
avgClass = ("UVAv" + band)[0:6] # Averaged data AIPS class
outIClass = parms["outIClass"] # image AIPS class
outCClass = parms["outCClass"] # image AIPS class
# Load the outputs pickle jar
ALMAFetchOutFiles()
# Logging directly to logFile
OErr.PInit(err, parms["prtLv"], logFile)
OSystem.PAllowThreads(nThreads) # Allow threads in Obit/oython
retCode = 0
ALMAAddOutFile(logFile, 'project', 'Pipeline log file')
mess = "Start project "+parms["project"]+" session "+parms["session"]+\
" "+parms["band"]+" Band"+" AIPS user no. "+str(AIPS.userno)+\
", ALMA Max. baseline "+str(parms["ALMAMaxBl"])
printMess(mess, logFile)
if debug:
pydoc.ttypager = pydoc.plainpager # don't page task input displays
mess = "Using Debug mode "
printMess(mess, logFile)
if check:
mess = "Only checking script"
printMess(mess, logFile)
# Log parameters
if parms['doLogParms']:
printMess("Parameter settings", logFile)
for p in parms:
mess = " " + p + ": " + str(parms[p])
printMess(mess, logFile)
# Save parameters to pickle jar, manifest
ParmsPicklefile = project + "_" + session + "_" + band + ".Parms.pickle" # Where results saved
SaveObject(parms, ParmsPicklefile, True)
ALMAAddOutFile(ParmsPicklefile, 'project', 'Processing parameters used')
# Are we going to be doing Hanning?
if parms["doHann"]:
loadClass = parms["band"] + "Raw"
else:
loadClass = dataClass
# Load Data from Archive directory
if parms["doLoadArchive"]:
uv = ALMAUVLoadArch(parms["archRoots"], ALMAAIPSName(project, session), loadClass, disk, parms["seq"], err, \
selConfig=parms["selConfig"], selBand=parms["selBand"], selChan=parms["selChan"], \
selChBW=parms["selChBW"], selNIF=parms["selNIF"], calInt=parms["calInt"], \
logfile=logFile, Compress=parms["Compress"], check=check, debug=debug)
if uv == None and not check:
raise RuntimeError, "Cannot load " + parms["archRoots"]
# Hanning
if parms["doHann"]:
# Set uv if not done
if uv == None and not check:
uv = UV.newPAUV("AIPS UV DATA", ALMAAIPSName(project, session),
loadClass[0:6], disk, parms["seq"], True, err)
if err.isErr:
OErr.printErrMsg(err, "Error creating AIPS data")
uv = ALMAHann(uv, ALMAAIPSName(project, session), dataClass, disk, parms["seq"], err, \
doDescm=parms["doDescm"], logfile=logFile, check=check, debug=debug)
if uv == None and not check:
raise RuntimeError, "Cannot Hann data "
# Set uv if not done
if uv == None and not check:
uv = UV.newPAUV("AIPS UV DATA", ALMAAIPSName(project, session), dataClass[0:6], \
disk, parms["seq"], True, err)
if err.isErr:
OErr.printErrMsg(err, "Error creating AIPS data")
# Clear any old calibration/editing
if parms["doClearTab"]:
mess = "Clear previous calibration"
printMess(mess, logFile)
ALMAClearCal(uv,
err,
doGain=parms["doClearGain"],
doFlag=parms["doClearFlag"],
doBP=parms["doClearBP"],
check=check)
OErr.printErrMsg(err, "Error resetting calibration")
# Copy FG 1 to FG 2
if parms["doCopyFG"]:
mess = "Copy FG 1 to FG 2"
printMess(mess, logFile)
retCode = ALMACopyFG(uv,
err,
logfile=logFile,
check=check,
debug=debug)
if retCode != 0:
raise RuntimeError, "Error Copying FG table"
# Drop end channels of spectra? Only if new FG 2
if parms["doCopyFG"] and (parms["BChDrop"] > 0) or (parms["EChDrop"] > 0):
# Channels based on original number, reduced if Hanning
nchan = uv.Desc.Dict["inaxes"][uv.Desc.Dict["jlocf"]]
fact = max(1, parms["selChan"] / nchan) # Hanning reduction factor
BChDrop = parms["BChDrop"] / fact
EChDrop = parms["EChDrop"] / fact
mess = "Trim %d channels from start and %d from end of each spectrum" % (
BChDrop, EChDrop)
printMess(mess, logFile)
retCode = ALMADropChan (uv, BChDrop, EChDrop, err, flagVer=parms["editFG"], \
logfile=logFile, check=check, debug=debug)
if retCode != 0:
raise RuntimeError, "Error Copying FG table"
# Get list of source in data
AllSource = ALMAAllSource(uv,
err,
logfile=logFile,
check=check,
debug=debug)
# Edit source lists to remove sources not present
for c in parms["PCInsCals"]:
if c not in AllSource:
parms["PCInsCals"].remove(c)
for t in parms["targets"]:
if t not in AllSource:
parms["targets"].remove(t)
if parms["XYGainSource"] not in AllSource:
parms["XYGainSource"] = None
if parms["XYDelaySource"] not in AllSource:
parms["XYDelaySource"] = None
for c in parms["DCals"]:
if c['Source'] not in AllSource:
c['Source'] = None
for c in parms["BPCals"]:
if c['Source'] not in AllSource:
c['Source'] = None
for c in parms["PCals"]:
if c['Source'] not in AllSource:
c['Source'] = None
for c in parms["ACals"]:
if c['Source'] not in AllSource:
c['Source'] = None
# Special editing
if parms["doEditList"] and not check:
mess = "Special editing"
printMess(mess, logFile)
for edt in parms["editList"]:
UV.PFlag(uv,err,timeRange=[dhms2day(edt["timer"][0]),dhms2day(edt["timer"][1])], \
flagVer=parms["editFG"], Ants=edt["Ant"], Chans=edt["Chans"], IFs=edt["IFs"], \
Stokes=edt["Stokes"], Reason=edt["Reason"])
OErr.printErrMsg(err, "Error Flagging")
# Quack to remove data from start and end of each scan
if parms["doQuack"]:
retCode = ALMAQuack (uv, err, begDrop=parms["quackBegDrop"], endDrop=parms["quackEndDrop"], \
Reason=parms["quackReason"], \
logfile=logFile, check=check, debug=debug)
if retCode != 0:
raise RuntimeError, "Error Quacking data"
# Flag antennas shadowed by others?
if parms["doShad"]:
retCode = ALMAShadow (uv, err, shadBl=parms["shadBl"], \
logfile=logFile, check=check, debug=debug)
if retCode != 0:
raise RuntimeError, "Error Shadow flagging data"
# Apply online calibration
if parms["doOnlineCal"]:
retCode = ALMAOnlineCal(uv,
err,
logfile=logFile,
check=check,
debug=debug)
if retCode != 0:
raise RuntimeError, "Error applying online calibration"
# Median window time editing, for RFI impulsive in time
if parms["doMedn"]:
mess = "Median window time editing, for RFI impulsive in time:"
printMess(mess, logFile)
retCode = ALMAMedianFlag (uv, " ", err, noScrat=noScrat, nThreads=nThreads, \
avgTime=parms["avgTime"], avgFreq=parms["avgFreq"], chAvg= parms["chAvg"], \
timeWind=parms["timeWind"], flagVer=2,flagSig=parms["mednSigma"], \
logfile=logFile, check=check, debug=False)
if retCode != 0:
raise RuntimeError, "Error in MednFlag"
# Median window frequency editing, for RFI impulsive in frequency
if parms["doFD1"]:
mess = "Median window frequency editing, for RFI impulsive in frequency:"
printMess(mess, logFile)
retCode = ALMAAutoFlag (uv, " ", err, flagVer=2, doCalib=-1, doBand=-1, \
timeAvg=parms["FD1TimeAvg"], \
doFD=True, FDmaxAmp=1.0e20, FDmaxV=1.0e20, FDwidMW=parms["FD1widMW"], \
FDmaxRMS=[1.0e20,0.1], FDmaxRes=parms["FD1maxRes"], \
FDmaxResBL= parms["FD1maxRes"], \
nThreads=nThreads, logfile=logFile, check=check, debug=debug)
if retCode != 0:
raise RuntimeError, "Error in AutoFlag"
# RMS/Mean editing for calibrators
if parms["doRMSAvg"]:
mess = "RMS/Mean editing for calibrators:"
printMess(mess, logFile)
clist = ALMACombineCals(parms["ACals"], parms["PCals"],
parms["DCals"]) # Calibrator list
retCode = ALMAAutoFlag (uv, clist, err, flagVer=2, doCalib=-1, doBand=-1, \
RMSAvg=parms["RMSAvg"], timeAvg=parms["RMSTimeAvg"], \
nThreads=nThreads, logfile=logFile, check=check, debug=debug)
if retCode != 0:
raise RuntimeError, "Error in AutoFlag"
# Need to find a reference antenna? See if we have saved it?
if (parms["refAnt"] <= 0):
refAnt = FetchObject(project + "_" + session + "_" + band +
".refAnt.pickle")
if refAnt:
parms["refAnt"] = refAnt
# Use bandpass calibrator and center half of each spectrum
if parms["refAnt"] <= 0:
mess = "Find best reference antenna: run Calib on BP Cal(s) "
printMess(mess, logFile)
parms["refAnt"] = ALMAGetRefAnt(uv, parms["BPCals"], err, flagVer=2, \
solInt=parms["bpsolint1"], nThreads=nThreads, \
logfile=logFile, check=check, debug=debug)
if err.isErr:
raise RuntimeError, "Error finding reference antenna"
if parms["refAnts"][0] <= 0:
parms["refAnts"][0] = parms["refAnt"]
mess = "Picked reference antenna " + str(parms["refAnt"])
printMess(mess, logFile)
# Save it
ParmsPicklefile = project + "_" + session + "_" + band + ".Parms.pickle" # Where results saved
SaveObject(parms, ParmsPicklefile, True)
refAntPicklefile = project + "_" + session + "_" + band + ".refAnt.pickle" # Where results saved
SaveObject(parms["refAnt"], refAntPicklefile, True)
# Plot Raw, edited data?
if parms["doRawSpecPlot"] and parms["plotSource"]:
mess = "Raw Spectral plot for: " + parms["plotSource"]
printMess(mess, logFile)
plotFile = "./" + fileRoot + "RawSpec.ps"
retCode = ALMASpectrum(uv, parms["plotSource"], parms["plotTime"], plotFile, parms["refAnt"], err, \
flagVer=2, Stokes=["XX","YY"], doband=-1, \
check=check, debug=debug, logfile=logFile )
if retCode != 0:
raise RuntimeError, "Error in Plotting spectrum"
ALMAAddOutFile(plotFile, 'project', 'Pipeline log file')
# delay calibration
if parms["doDelayCal"] and parms["DCals"] and not check:
plotFile = "./" + fileRoot + "DelayCal.ps"
retCode = ALMADelayCal(uv, parms["DCals"], err, \
BChan=parms["delayBChan"], EChan=parms["delayEChan"], \
doCalib=2, flagVer=2, doBand=-1, \
solInt=parms["delaySolInt"], smoTime=parms["delaySmoo"], \
refAnts=[parms["refAnt"]], doTwo=parms["doTwo"],
doZeroPhs=parms["delayZeroPhs"], \
doPlot=parms["doSNPlot"], plotFile=plotFile, \
nThreads=nThreads, noScrat=noScrat, \
logfile=logFile, check=check, debug=debug)
if retCode != 0:
raise RuntimeError, "Error in delay calibration"
# Plot corrected data?
if parms["doSpecPlot"] and parms["plotSource"]:
plotFile = "./" + fileRoot + "DelaySpec.ps"
retCode = ALMASpectrum(uv, parms["plotSource"], parms["plotTime"], \
plotFile, parms["refAnt"], err, \
flagVer=2, Stokes=["XX","YY"], doband=-1, \
check=check, debug=debug, logfile=logFile )
if retCode != 0:
raise RuntimeError, "Error in Plotting spectrum"
# Bandpass calibration
if parms["doBPCal"] and parms["BPCals"]:
plotFile = "./" + fileRoot + "BPCal.ps"
retCode = ALMABPCal(uv, parms["BPCals"], err, noScrat=noScrat, solInt1=parms["bpsolint1"], \
solInt2=parms["bpsolint2"], solMode=parms["bpsolMode"], \
BChan1=parms["bpBChan1"], EChan1=parms["bpEChan1"], \
BChan2=parms["bpBChan2"], EChan2=parms["bpEChan2"], ChWid2=parms["bpChWid2"], \
doCenter1=parms["bpDoCenter1"], refAnt=parms["refAnt"], \
UVRange=parms["bpUVRange"], doCalib=2, gainUse=0, flagVer=2, doPlot=False, \
doAmpEdit=parms["doAmpEdit"], ampSigma=parms["ampSigma"], \
doBPPlot=parms["doBPPlot"], plotBPFile=plotFile, \
nThreads=nThreads, logfile=logFile, check=check, debug=debug)
if retCode != 0:
raise RuntimeError, "Error in Bandpass calibration"
# Plot corrected data?
if parms["doSpecPlot"] and parms["plotSource"]:
plotFile = "./" + fileRoot + "BPSpec.ps"
retCode = ALMASpectrum(uv, parms["plotSource"], parms["plotTime"], plotFile, \
parms["refAnt"], err, flagVer=2, Stokes=["XX","YY"], doband=1, \
check=check, debug=debug, logfile=logFile )
if retCode != 0:
raise RuntimeError, "Error in Plotting spectrum"
# set X/Y gains and initial calibration
if parms["doXYFixGain"] and parms["XYGainSource"] and not check:
mess = "Fix X/Y gain ratios"
printMess(mess, logFile)
retCode = ALMAXYGain(uv, err, \
XYCal=parms["XYGainSource"],timerange=parms["XYGainTime"], \
doCalib=2, gainUse=0, doBand=1, flagVer=2, refAnt=parms["refAnt"], \
nThreads=nThreads, noScrat=noScrat, logfile=logFile, \
check=check, debug=debug)
if retCode != 0:
raise RuntimeError, "Error in X-Y gain fix"
# Self calibrate calibrators
if parms["doImgCal"] and not check:
mess = "SelfCalibrate/Image calibrators"
printMess(mess, logFile)
src = ALMACombineCals(parms["ACals"], parms["PCals"],
parms["APCals"]) # Source list
ALMAImageCals(uv, err, Sources=src, seq=parms["seq"], \
sclass=parms["outCClass"], doCalib=2, flagVer=2, doBand=1, FOV=parms["CalFOV"], \
maxPSCLoop=parms["maxPSCLoop"], minFluxPSC=parms["minFluxPSC"], solPInt=parms["solPInt"], \
maxASCLoop=parms["maxASCLoop"], minFluxASC=parms["minFluxASC"],\
solAInt=parms["solAInt"], avgPol=parms["avgPol"], Niter=1000, \
avgIF=parms["avgIF"], minSNR=parms["minSNR"],\
refAnt=parms["refAnt"], nThreads=nThreads, noScrat=noScrat,\
logfile=logFile, check=check, debug=debug)
# Self calibrated models now available
ALMAImageModel(parms["ACals"], parms["outCClass"], disk, parms["seq"], err)
ALMAImageModel(parms["PCals"], parms["outCClass"], disk, parms["seq"], err)
ALMAImageModel(parms["APCals"], parms["outCClass"], disk, parms["seq"],
err)
# Phase Calibrate
if parms["doPhaseCal"]:
plotFile = "./" + fileRoot + "PhaseCal.ps"
retCode = ALMAPhaseCal (uv, parms["PCals"], err, ACals=parms["ACals"], \
doCalib=2, doBand=1, BPVer=1, flagVer=2, refAnt=parms["refAnt"], \
BChan=parms["ampBChan"], EChan=parms["ampEChan"], \
solInt=parms["solPInt"], avgIF = parms['avgIF'], \
ampScalar=parms["ampScalar"], doPlot=parms["doSNPlot"], plotFile=plotFile, \
nThreads=nThreads, noScrat=noScrat, logfile=logFile, check=check, debug=debug)
if retCode != 0:
raise RuntimeError, "Error calibrating"
# Amp & phase Calibrate
if parms["doAmpPhaseCal"]:
plotFile = "./" + fileRoot + "APCal.ps"
retCode = ALMACalAP (uv, [], parms["ACals"], err, PCals=parms["APCals"], \
doCalib=2, doBand=1, BPVer=1, flagVer=2, \
BChan=parms["ampBChan"], EChan=parms["ampEChan"], \
solInt=parms["solAInt"], solSmo=parms["solSmo"], ampScalar=parms["ampScalar"], \
doAmpEdit=parms["doAmpEdit"], ampSigma=parms["ampSigma"], \
ampEditFG=parms["ampEditFG"], \
doPlot=parms["doSNPlot"], plotFile=plotFile, refAnt=parms["refAnt"], \
nThreads=nThreads, noScrat=noScrat, logfile=logFile, check=check, debug=debug)
if retCode != 0:
raise RuntimeError, "Error calibrating"
# More editing
if parms["doAutoFlag"]:
mess = "Post calibration editing:"
printMess(mess, logFile)
clist = []
retCode = ALMAAutoFlag (uv, clist, err, flagVer=2, \
doCalib=2, gainUse=0, doBand=1, BPVer=1, \
IClip=parms["IClip"], minAmp=parms["minAmp"], timeAvg=parms["timeAvg"], \
doFD=parms["doAFFD"], FDmaxAmp=parms["FDmaxAmp"], FDmaxV=parms["FDmaxV"], \
FDwidMW=parms["FDwidMW"], FDmaxRMS=parms["FDmaxRMS"], \
FDmaxRes=parms["FDmaxRes"], FDmaxResBL=parms["FDmaxResBL"], \
FDbaseSel=parms["FDbaseSel"], \
nThreads=nThreads, logfile=logFile, check=check, debug=debug)
if retCode != 0:
raise RuntimeError, "Error in AutoFlag"
# Calibrate and average data
if parms["doCalAvg"]:
retCode = ALMACalAvg (uv, avgClass, parms["seq"], parms["CalAvgTime"], err, \
flagVer=2, doCalib=2, gainUse=0, doBand=1, BPVer=1, doPol=False, \
avgFreq=parms["CAavgFreq"], chAvg=parms["CAchAvg"], \
BChan=parms["CABChan"], EChan=parms["CAEChan"], \
BIF=parms["CABIF"], EIF=parms["CAEIF"], Compress=parms["Compress"], \
nThreads=nThreads, logfile=logFile, check=check, debug=debug)
if retCode != 0:
raise RuntimeError, "Error in CalAvg"
# Get calibrated/averaged data
if not check:
try:
uv = UV.newPAUV("AIPS UV DATA", ALMAAIPSName(project, session), avgClass[0:6], \
disk, parms["seq"], True, err)
except Exception, exception:
mess = "No Averaged/calibrated datafile"
printMess(mess, logFile)
return
def VLAPolCal(uv, InsCals, RLCal, RLPhase, err, RM=0.0, \
doCalib=2, gainUse=0, flagVer=-1, \
soltype="APPR", fixPoln=False, avgIF=False, \
solInt=0.0, refAnt=0, \
pmodel=[0.0,0.0,0.0,0.0,0.0,0.0,0.0], \
FOV=0.05, niter = 100, \
nThreads=1, noScrat=[], logfile = ""):
""" Polarization calibration, both instrumental and orientation
Do Instrumental and R-L calibration
Instrumental cal uses PCAL, R-L cal is done by imaging each IF in Q and U
and summing the CLEAN components.
uv = UV data object to calibrate
InsCals = Instrumental poln calibrators, name or list of names
If None no instrumental cal
RLCal = R-L (polarization angle) calibrator,
If None no R-L cal
RLPhase = R-L phase of RLCal (deg) at 1 GHz
err = Obit error/message stack
RM = Rotation measure of RLCal
doCalib = Apply prior calibration table, positive=>calibrate
gainUse = CL/SN table to apply
flagVer = Input Flagging table version
soltype = solution type
fixPoln = if True, don't solve for source polarization in ins. cal
avgIF = if True, average IFs in ins. cal.
solInt = instrumental solution interval (min), 0=> scan average
refAnt = Reference antenna
pmodel = Instrumental poln cal source poln model.
pmodel[0] = I flux density (Jy)
pmodel[1] = Q flux density (Jy)
pmodel[2] = U flux density (Jy)
pmodel[3] = V flux density (Jy)
pmodel[4] = X offset in sky (arcsec)
pmodel[5] = Y offset in sky (arcsec)
FOV = field of view radius (deg) needed to image RLCal
niter = Number of iterations of CLEAN in R-L cal
nThreads = Number of threads to use in imaging
noScrat = list of disks to avoid for scratch files
logfile = Log file for task
"""
################################################################
# Instrumental calibrtation
if InsCals != None:
pcal = AIPSTask.AIPSTask("pcal")
pcal.logFile = logfile
setname(uv, pcal)
if type(InsCals) == list:
pcal.calsour[1:] = InsCals
else:
pcal.calsour[1:] = [InsCals]
pcal.docalib = doCalib
pcal.gainuse = gainUse
pcal.flagver = flagVer
pcal.soltype = soltype
pcal.solint = solInt
pcal.refant = refAnt
if fixPoln:
pcal.bparm[10] = 1.0
if avgIF:
pcal.cparm[1] = 1.0
pcal.pmodel[1:] = pmodel
pcal.i
pcal.g
# end instrumental poln cal
# R-L phase cal
if RLCal != None:
img = ObitTask.ObitTask("Imager")
img.logFile = logfile
setname(uv, img)
img.doCalib = doCalib
img.gainUse = gainUse
img.flagVer = flagVer
img.doPol = True
img.Sources[0] = RLCal
img.Stokes = "IQU"
img.FOV = FOV
img.Niter = niter
img.autoWindow = True
img.dispURL = "None"
img.Catalog = "None"
img.nThreads = nThreads
img.noScrat = noScrat
# Temporary output files
if img.DataType == "AIPS":
img.outName = "TEMP"
img.outClass = "IPOLCL"
img.outDisk = img.inDisk
img.outSeq = 6666
img.out2Name = "TEMP"
img.out2Class = "IPOLCL"
img.out2Disk = img.inDisk
img.out2Seq = 7777
elif img.DataType == "FITS":
img.outFile = "TEMPPOLCAL.fits"
img.outDisk = img.inDisk
img.out2File = "TEMPPOLCAL2.uvtab"
img.out2Disk = img.inDisk
# How many IFs?
h = uv.Desc.Dict
if h["jlocif"] >= 0:
nif = h["inaxes"][h["jlocif"]]
else:
nif = 1
# Lists of flux densities and RMSes
IFlux = []
IRMS = []
QFlux = []
QRMS = []
UFlux = []
URMS = []
# Loop over IF imaging I,Q, U
for iif in range(1, nif + 1):
img.BIF = iif
img.EIF = iif
#img.dispURL = "ObitView" # DEBUG
#img.debug=True # DEBUG
img.g
# Get fluxes from inner quarter of images
if img.DataType == "AIPS":
outName = (img.Sources[0].strip() + "TEMP")[0:12]
outDisk = img.outDisk
outSeq = 6666
# Stokes I
outClass = "IPOLCL"
x = Image.newPAImage("I", outName, outClass, outDisk, outSeq,
True, err)
h = x.Desc.Dict
blc = [h["inaxes"][0] / 4, h["inaxes"][1] / 4]
trc = [3 * h["inaxes"][0] / 4, 3 * h["inaxes"][1] / 4]
stat = imstat(x, err, blc=blc, trc=trc)
IFlux.append(stat["Flux"])
IRMS.append(stat["RMSHist"])
x.Zap(err) # Cleanup
del x
# Stokes Q
outClass = "QPOLCL"
x = Image.newPAImage("Q", outName, outClass, outDisk, outSeq,
True, err)
stat = imstat(x, err, blc=blc, trc=trc)
QFlux.append(stat["Flux"])
QRMS.append(stat["RMSHist"])
x.Zap(err) # Cleanup
del x
# Stokes U
outClass = "UPOLCL"
x = Image.newPAImage("U", outName, outClass, outDisk, outSeq,
True, err)
stat = imstat(x, err, blc=blc, trc=trc)
UFlux.append(stat["Flux"])
URMS.append(stat["RMSHist"])
x.Zap(err) # Cleanup
del x
# Delete UV output
out2Name = (img.Sources[0].strip() + "TEMP")[0:12]
out2Class = "IPOLCL"
out2Disk = img.inDisk
out2Seq = 7777
u = UV.newPAUV("UV", out2Name, out2Class, out2Disk, out2Seq,
True, err)
u.Zap(err)
del u
elif img.DataType == "FITS":
# Stokes I
outFile = img.Sources[0].strip() + "ITEMPPOLCAL.fits"
x = Image.newPFImage("I", outFile, img.outDisk, True, err)
h = x.Desc.Dict
blc = [h["inaxes"][0] / 4, h["inaxes"][1] / 4]
trc = [3 * h["inaxes"][0] / 4, 3 * h["inaxes"][1] / 4]
stat = imstat(x, err, blc=blc, trc=trc)
IFlux.append(stat["Flux"])
IRMS.append(stat["RMSHist"])
x.Zap(err) # Cleanup
del x
# Stokes Q
outFile = img.Sources[0].strip() + "ITEMPPOLCAL.fits"
x = Image.newPFImage("Q", outFile, img.outDisk, True, err)
stat = imstat(x, err, blc=blc, trc=trc)
IFlux.append(stat["Flux"])
IRMS.append(stat["RMSHist"])
x.Zap(err) # Cleanup
del x
# Stokes U
outFile = img.Sources[0].strip() + "ITEMPPOLCAL.fits"
x = Image.newPFImage("Q", outFile, img.outDisk, True, err)
stat = imstat(x, err, blc=blc, trc=trc)
IFlux.append(stat["Flux"])
IRMS.append(stat["RMSHist"])
x.Zap(err) # Cleanup
del x
out2File = img.Sources[0].strip() + "TEMPPOLCAL2.uvtab"
u = UV.newPFUV("UV", outFile, img.outDisk, True, err)
u.Zap(err)
del u
# End accumulate statistics by file type
# End loop over IF
# Give results, compute R-L correction
RLCor = []
import math
mess = " IF IFlux IRMS QFlux QRMS UFlux URMS R-L Corr"
printMess(mess, logfile)
for i in range(0, len(IFlux)):
# REALLY NEED RM Correction!!!!!
cor = RLPhase - 57.296 * math.atan2(UFlux[i], QFlux[i])
RLCor.append(cor)
mess = "%3d %8.3f %8.3f %7.3f %7.3f %7.3f %7.3f %7.3f "%\
(i+1, IFlux[i], IRMS[i], QFlux[i], QRMS[i], UFlux[i], URMS[i], cor)
printMess(mess, logfile)
# Copy highest CL table
hiCL = uv.GetHighVer("AIPS CL")
# Apply R-L phase corrections
clcor = AIPSTask.AIPSTask("clcor")
clcor.logFile = logfile
setname(uv, clcor)
clcor.opcode = "POLR"
clcor.gainver = hiCL
clcor.gainuse = hiCL + 1
clcor.clcorprm[1:] = RLCor
clcor.g
def pipeline(aipsSetup, parmFile):
"""
VLBA Continuum pipeline.
* *aipsSetup* = AIPS setup file
* *parmFile* = pipeline input parameters file
"""
############################# Initialize OBIT ##########################################
noScrat = []
exec(open(aipsSetup).read())
VLBAAddOutFile(aipsSetup, 'project', "Obit's AIPS setup file")
############################# Default parameters ##########################################
# Define data
project = "Unspecified" # Project name (12 char or less, used as AIPS Name)
session = "?" # Project session code
band = "?" # Observing band
logFile = project + "_" + session + "_" + band + ".log" # Processing log file
seq = 1 # AIPS sequence number
gain = 0.10 # CLEAN loop gain
doLoadIDI = True # Load data from IDI FITS?
doLoadUVF = False # Load the "AIPS Friendly" (uvfits) FITS version
dataInUVF = None # Input uvfits data file name
dataInIDI = None # Input FITS-IDI file or list
dataClass = "Raw" # AIPS class of raw uv data
Compress = False # Use compressed UV data?
calInt = 0.15 # Calibration table interval in min.
wtThresh = 0.8 # Data weight threshold
check = False # Only check script, don't execute tasks
debug = False # run tasks debug
prtLv = 2 # Print level
# Initialize parameters
parms = VLBAInitContParms()
############################# Set Project Processing parameters ##################
exec(open(parmFile).read())
VLBAAddOutFile(parmFile, 'project', 'Pipeline input parameters')
################################## Process #####################################
# Init cal pickle jars
goodCalPicklefile = project + "_" + session + "_" + band + "_GoodCal.pickle" # Where results saved
# Default "best" calibration
goodCal = {
"Source": " ",
"souID": 0,
"timeRange": (0.0, 100.0),
"Fract": 0.0,
"SNR": 0.0,
"bestRef": 0
}
# Save initial default if it doesn't already exist
SaveObject(goodCal, goodCalPicklefile, False)
VLBAAddOutFile(goodCalPicklefile, 'project', 'Best calibrator information')
OKCalPicklefile = project + "_" + session + "_" + band + "_OKCal.pickle" # Where results saved
SaveObject(parms["contCals"], OKCalPicklefile, False)
VLBAAddOutFile(OKCalPicklefile, 'project', 'List of calibrators')
targetsPicklefile = project + "_" + session + "_" + band + "_targets.pickle" # Where results saved
if (not parms["targets"]) and os.path.exists(targetsPicklefile):
parms["targets"] = FetchObject(targetsPicklefile)
else:
SaveObject(parms["targets"], targetsPicklefile, False)
VLBAAddOutFile(targetsPicklefile, 'project', 'List of targets')
# Load the outputs pickle jar
VLBAFetchOutFiles()
# Logging directly to logFile
OErr.PInit(err, prtLv, logFile)
retCode = 0
VLBAAddOutFile(logFile, 'project', 'Pipeline log file')
mess = "Start project " + project + " session " + session + " " + band + " Band" + " AIPS user no. " + str(
AIPS.userno)
printMess(mess, logFile)
logger.info(mess)
if debug:
pydoc.ttypager = pydoc.plainpager # don't page task input displays
mess = "Using Debug mode "
printMess(mess, logFile)
if check:
mess = "Only checking script"
printMess(mess, logFile)
# Load Data from FITS
uv = None # Raw data
uvc = None # Cal/averaged data
if doLoadIDI:
logger.info("--> Load IDI data file (doLoadIDI)")
if type(dataInIDI) == list:
logger.info(
"--> Loading a list of IDI files (dataInIDI is a list)\n" +
" This data appears to be from the old correlator.\n" +
" IDI corrections will be applied when loading is\n" +
" complete.")
# Read list
for dataIn in dataInIDI:
uv = VLBAIDILoad(dataIn, project, session, band, dataClass, disk, seq, err, logfile=logFile, \
wtThresh=wtThresh, calInt=calInt, Compress=Compress, \
check=check, debug=debug)
if not UV.PIsA(uv):
raise RuntimeError("Cannot load " + dataIn)
# Fix IDI files:
uv = IDIFix.IDIFix(uv, err)
seq = uv.Aseq
else:
logger.info(
"--> Loading a single IDI file (dataInIDI is not a list)\n" +
" This data appears to be from the DiFX correlator.\n" +
" No IDI corrections will be applied.")
# Single IDI file
uv = VLBAIDILoad(dataInIDI, project, session, band, dataClass, disk, seq, err, logfile=logFile, \
wtThresh=wtThresh, calInt=calInt, Compress=Compress, \
check=check, debug=debug)
if not UV.PIsA(uv):
raise RuntimeError("Cannot load " + dataInIDI)
if doLoadUVF:
logger.info("--> Load UVFITS data file (doLoadUVF)")
uv = VLBAIDILoad(dataInUVF, project, session, band, dataClass, disk, seq, err, logfile=logFile, \
wtThresh=wtThresh, calInt=calInt, Compress=Compress, \
check=check, debug=debug)
# Adding check condition to avoid error when checking
if not UV.PIsA(uv) and not check:
raise RuntimeError("Cannot load " + dataInUVF)
# Otherwise set uv
if uv == None and not check:
Aname = VLBAAIPSName(project, session)
uvname = project + "_" + session + "_" + band
uv = UV.newPAUV(uvname, Aname, dataClass, disk, seq, True, err)
if err.isErr:
OErr.printErrMsg(err, "Error creating AIPS data")
# frequency dependent default parameters
VLBAInitContFQParms(uv, parms, err, \
logfile=logFile, check=check, debug=debug)
# Setup target source list
if os.path.exists(targetsPicklefile):
parms["targets"] = \
setupSourceList( parms["targets"], uv, err, logFile, check, debug )
logger.debug("parms['targets'] = " + str(parms["targets"]))
# Clear any old calibration/editing
if parms["doClearTab"]:
logger.info("--> Clear old calibration/editing (doClearTab)")
VLBAClearCal(uv, err, doGain=parms["doGain"], doFlag=parms["doFlag"], doBP=parms["doBP"], \
check=check, logfile=logFile)
OErr.printErrMsg(err, "Error resetting calibration")
# Copy FG 1 to FG 2
if parms["doCopyFG"]:
logger.info("--> Copy flag (FG) table 1 to flag table 2 (doCopyFG)")
retCode = VLBACopyFG(uv,
err,
logfile=logFile,
check=check,
debug=debug)
if retCode != 0:
raise RuntimeError("Error Copying FG table")
# Special editing
if parms["doEditList"] and not check:
logger.info("--> Special editing (doEditList)")
for edt in parms["editList"]:
UV.PFlag(uv,err,timeRange=[dhms2day(edt["timer"][0]),dhms2day(edt["timer"][1])], \
flagVer=editFG, Ants=edt["Ant"], Chans=edt["Chans"], IFs=edt["IFs"], \
Stokes=edt["Stokes"], Reason=edt["Reason"])
OErr.printErrMsg(err, "Error Flagging")
# Quack to remove data from start and end of each scan
if parms["doQuack"]:
logger.info("--> Remove data from start and end of scans (doQuack)")
retCode = VLBAQuack (uv, err, \
begDrop=parms["quackBegDrop"], endDrop=parms["quackEndDrop"], \
Reason=parms["quackReason"], \
logfile=logFile, check=check, debug=debug)
if retCode != 0:
raise RuntimeError("Error Quacking data")
# Median window time editing, for RFI impulsive in time or dropouts
if parms["doMedn"]:
logger.info("--> Median window time editing (doMedn)")
retCode = VLBAMedianFlag (uv, " ", err, noScrat=noScrat, nThreads=nThreads, \
avgTime=parms["mednAvgTime"], avgFreq=parms["mednAvgFreq"], \
chAvg=parms["mednChAvg"], timeWind=parms["mednTimeWind"], \
flagSig=parms["mednSigma"], flagVer=2, \
logfile=logFile, check=check, debug=False)
if retCode != 0:
raise RuntimeError("Error in MednFlag")
# Quantization correction?
if parms["doQuantCor"]:
logger.info("--> Quantization correction (doQuantCor)")
plotFile = project + "_" + session + "_" + band + ".Quant.ps"
retCode = VLBAQuantCor(uv, parms["QuantSmo"], parms["QuantFlag"], err, \
doSNPlot=parms["doSNPlot"], plotFile=plotFile, \
logfile=logFile, check=check, debug=debug)
if retCode != 0:
raise RuntimeError("Error in quantization correcting/flagging")
# Parallactic angle correction?
if parms["doPACor"]:
logger.info("--> Parallactic angle correction (doPACor)")
retCode = VLBAPACor(uv, err, noScrat=noScrat, \
logfile=logFile, check=check, debug=debug)
if retCode != 0:
raise RuntimeError("Error in quantization correcting/flagging")
# Opacity/Tsys/gain correction
if parms["doOpacCor"]:
logger.info("--> Opacity/Tsys/Gain correction (doOpacCor)")
plotFile = project + "_" + session + "_" + band + ".Opacity.ps"
retCode = VLBAOpacCor(uv, parms["OpacSmoo"], err, \
doSNPlot=parms["doSNPlot"], plotFile=plotFile, \
logfile=logFile, check=check, debug=debug)
if retCode != 0:
raise RuntimeError("Error in opacity/gain.Tsys correction")
VLBASaveOutFiles() # Save plot file in Outfiles
# Need to determine a list of calibrators?
if (parms["contCals"] == None) or (len(parms["contCals"]) <= 0):
logger.info("--> Get list of calibrators (contCals = None or [])")
if parms["doFindOK"]:
slist = VLBAAllSource(uv,
err,
logfile=logFile,
check=check,
debug=debug)
parms["contCals"] = VLBAOKCal(uv, parms["minOKFract"], err, \
solInt=parms["findSolInt"], \
calSou=slist, minSNR=parms["minOKSNR"], \
doCalib=-1, flagVer=2, refAnts=parms["refAnts"], \
noScrat=noScrat, nThreads=nThreads, \
logfile=logFile, check=check, debug=debug)
if not parms["contCals"] and not check:
raise RuntimeError("Error in finding acceptable calibrators")
logger.info("Calibrators = " + str(parms["contCals"]))
else:
# Snatch from pickle jar
parms["contCals"] = FetchObject(OKCalPicklefile)
# Save contCals to a pickle jar
SaveObject(parms["contCals"], OKCalPicklefile, True)
# Find best calibration source
if parms["doFindCal"]:
logger.info("--> Find best calibration source (doFindCal)")
goodCal = VLBAGoodCal(uv, err, \
solInt=parms["findSolInt"], timeInt=parms["findTimeInt"], \
calSou=parms["contCals"], \
#CalModel=parms["contCalModel"], \
doCalib=-1, flagVer=2, refAnts=parms["refAnts"], \
noScrat=noScrat, nThreads=nThreads, \
logfile=logFile, check=check, debug=debug)
if not goodCal and not check:
raise RuntimeError("Error in finding best calibration data")
# Save it to a pickle jar
SaveObject(goodCal, goodCalPicklefile, True)
else:
# Fetch from pickle
goodCal = FetchObject(goodCalPicklefile)
# Apply Phase cals from PC table?
if parms["doPCcor"] and not check:
logger.info("--> Apply phase cals (doPCcor)")
plotFile = project + "_" + session + "_" + band + ".PC.ps"
retCode = VLBAPCcor(uv, err, calSou=goodCal["Source"], \
timeRange=goodCal["timeRange"], \
doCalib=-1, flagVer=2, solInt=parms["manPCsolInt"], \
PCin=1, SNout=0, refAnt=goodCal["bestRef"], \
doPCPlot=parms["doPCPlot"], plotFile=plotFile, \
noScrat=noScrat, logfile=logFile, check=check, debug=debug)
if retCode != 0:
raise RuntimeError("Error in PC calibration")
VLBASaveOutFiles() # Save plot file in Outfiles
# manual phase cal
if parms["doManPCal"] and not check:
logger.info("--> Manual phase cal (doManPCal)")
plotFile = project + session + band + ".ManPCal.ps"
retCode = VLBAManPCal(uv, err, calSou=goodCal["Source"], \
#CalModel=parms["contCalModel"], \
timeRange=goodCal["timeRange"], \
solInt=parms["manPCsolInt"], smoTime=parms["manPCSmoo"], \
refAnts=[goodCal["bestRef"]], doCalib=2, flagVer=2, \
doManPCalPlot=parms["doManPCalPlot"], \
plotFile=plotFile, noScrat=noScrat, \
nThreads=nThreads, logfile=logFile, check=check, debug=debug)
if retCode != 0:
raise RuntimeError("Error in manual phase calibration")
# image cals
if parms["doImgCal"] and not check:
logger.info("--> Image calibrators (doImgCal)")
VLBAImageCals(uv,
err,
Sources=parms["contCals"],
seq=seq,
sclass=parms["outCclass"],
doCalib=2,
flagVer=2,
doBand=0,
FOV=parms["FOV"],
Robust=parms["Robust"],
maxPSCLoop=parms["maxPSCLoop"],
minFluxPSC=parms["minFluxPSC"],
solPInt=parms["solPInt"],
solMode=parms["solMode"],
maxASCLoop=parms["maxASCLoop"],
minFluxASC=parms["minFluxASC"],
solAInt=parms["solAInt"],
avgPol=parms["avgPol"],
avgIF=parms["avgIF"],
minSNR=parms["minSNR"],
refAnt=goodCal["bestRef"],
nThreads=nThreads,
noScrat=noScrat,
logfile=logFile,
check=check,
debug=debug)
# Rewrite OKCal pickle file because calibrators may have been updated
SaveObject(parms["contCals"], OKCalPicklefile, True)
if len(parms["contCals"]) <= 0:
logger.error(
"No calibration sources have been detected! Stopping pipeline."
)
raise RuntimeError("No calibration sources have been detected!")
# Check if calibrator models now available
parms["contCalModel"] = VLBAImageModel(parms["contCals"],
parms["outCclass"], disk, seq, err)
# delay calibration
if parms["doDelayCal"] and not check:
logger.info("--> Delay calibration (doDelayCal)")
plotFile = project + "_" + session + "_" + band + ".DelayCal.ps"
retCode = VLBADelayCal(uv, err, calSou=parms["contCals"], CalModel=parms["contCalModel"], \
doCalib=2, flagVer=2, doBand=0, \
solInt=parms["manPCsolInt"], smoTime=parms["delaySmoo"], \
refAnts=[goodCal["bestRef"]], \
doSNPlot=parms["doSNPlot"], plotFile=plotFile, \
nThreads=nThreads, noScrat=noScrat, logfile=logFile, check=check, debug=debug)
if retCode != 0:
raise RuntimeError("Error in delay calibration")
VLBASaveOutFiles() # Save plot file in Outfiles
# Bandpass calibration if needed
if parms["doBPCal"] and not check:
logger.info("--> Bandpass calibration (doBPCal)")
retCode = VLBABPass(uv, goodCal["Source"], err, CalModel=parms["contCalModel"], \
timeRange=goodCal["timeRange"], doCalib=2, flagVer=2, \
noScrat=noScrat, solInt1=parms["bpsolint1"], \
solInt2=parms["bpsolint2"], solMode=parms["bpsolMode"], \
BChan1=parms["bpBChan1"], EChan1=parms["bpEChan1"], BChan2=parms["bpBChan2"], \
EChan2=parms["bpEChan2"], ChWid2=parms["bpChWid2"], \
doCenter1=parms["bpDoCenter1"], refAnt=goodCal["bestRef"], specIndex=parms["specIndex"], \
doAuto = parms["bpdoAuto"], \
nThreads=nThreads, logfile=logFile, check=check, debug=debug)
if retCode != 0:
raise RuntimeError("Error in Bandpass calibration")
# Plot amplitude and phase vs. frequency
if parms["doSpecPlot"]:
logger.info("--> Spectral plotting (doSpecPlot)")
plotFile = project + '_' + session + '_' + band + ".spec.ps"
VLBASpecPlot(uv,
goodCal,
err,
doband=1,
check=check,
plotFile=plotFile,
logfile=logFile,
debug=debug)
VLBASaveOutFiles() # Save plot file in Outfiles
# Phase calibration using calibrator models
if parms["doPhaseCal"]:
logger.info(
"--> Phase calibration using calibrator models (doPhaseCal)")
plotFile = project + "_" + session + "_" + band + ".PhaseCal0.ps"
retCode = VLBAPhaseCal(uv, err, calSou=parms["contCals"], CalModel=parms["contCalModel"], \
doCalib=-1, flagVer=0, doBand=-1, \
refAnt=goodCal["bestRef"], solInt=parms["solPInt"], \
doSNPlot=parms["doSNPlot"], plotFile=plotFile, \
nThreads=nThreads, noScrat=noScrat, logfile=logFile, check=check, debug=debug)
if retCode != 0:
raise RuntimeError("Error in phase calibration")
VLBASaveOutFiles() # Save plot file in Outfiles
# Rewrite contCals pickle file because contCals may have been updated
SaveObject(parms["contCals"], OKCalPicklefile, True)
if len(parms["contCals"]) <= 0:
logger.error(
"No calibrator sources have been detected! Stopping pipeline.")
raise RuntimeError("No calibrator sources have been detected!")
# Amplitude calibration
if parms["doAmpCal"] and not check:
logger.info("--> Amplitude calibration (doAmpCal)")
plotFile = project + "_" + session + "_" + band + ".AmpCal.ps"
retCode = VLBAAmpCal(uv,
err,
calSou=parms["contCals"],
CalModel=parms["contCalModel"],
doCalib=2,
flagVer=2,
doBand=1,
minSumCC=parms["minFluxASC"],
refAnt=goodCal["bestRef"],
solInt=parms["solAInt"],
smoTimeA=2.0,
smoTimeP=1. / 60.,
doSNPlot=parms["doSNPlot"],
plotFile=plotFile,
nThreads=nThreads,
noScrat=noScrat,
logfile=logFile,
check=check,
debug=debug)
if retCode != 0:
raise RuntimeError("Error in amplitude calibration")
VLBASaveOutFiles() # Save plot file in Outfiles
# Rewrite contCals pickle file because contCals may have been updated
SaveObject(parms["contCals"], OKCalPicklefile, True)
if len(parms["contCals"]) <= 0:
logger.error(
"No calibrator sources have been detected! Stopping pipeline.")
raise RuntimeError("No calibrator sources have been detected!")
# Calibrate and average data
if parms["doCalAvg"]:
logger.info("--> Calibration and average data (doCalAvg)")
retCode = VLBACalAvg (uv, parms["avgClass"], seq, parms["CalAvgTime"], err, \
flagVer=2, doCalib=2, gainUse=0, doBand=1, BPVer=1, \
BIF=parms["CABIF"], EIF=parms["CAEIF"], \
BChan=parms["CABChan"], EChan=parms["CAEChan"], \
chAvg=parms["chAvg"], avgFreq=parms["avgFreq"], Compress=Compress, \
logfile=logFile, check=check, debug=debug)
if retCode != 0:
raise RuntimeError("Error in CalAvg")
# image targets phase only self-cal - NOTE: actually A&P
if parms["doImgTarget"] and not check:
logger.info("--> Image targets (doImgTargets)")
if not uvc:
# Get calibrated/averaged data
Aname = VLBAAIPSName(project, session)
uvname = project + "_" + session + "_" + band + "_Cal"
uvc = UV.newPAUV(uvname, Aname, parms["avgClass"], disk, seq, True,
err)
if err.isErr:
OErr.printErrMsg(err, "Error creating cal/avg AIPS data")
logger.debug("parms['targets'] = " + str(parms["targets"]))
VLBAImageCals(uv,
err,
Sources=parms["targets"],
seq=seq,
sclass=parms["outTclass"],
doCalib=2,
flagVer=2,
doBand=1,
FOV=parms["FOV"],
Robust=parms["Robust"],
maxPSCLoop=parms["maxPSCLoop"],
minFluxPSC=parms["minFluxPSC"],
solPInt=parms["solPInt"],
solMode=parms["solMode"],
maxASCLoop=parms["maxASCLoop"],
minFluxASC=parms["minFluxASC"],
solAInt=parms["solAInt"],
avgPol=parms["avgPol"],
avgIF=parms["avgIF"],
minSNR=parms["minSNR"],
refAnt=goodCal["bestRef"],
nThreads=nThreads,
noScrat=noScrat,
logfile=logFile,
check=check,
debug=debug)
# Rewrite targets pickle file because targets may have been updated
SaveObject(parms["targets"], targetsPicklefile, True)
if len(parms["targets"]) <= 0:
logger.error(
"No target sources have been detected! Stopping pipeline.")
raise RuntimeError("No target sources have been detected!")
# Phase calibration using target models
if parms["doPhaseCal"]:
logger.info("--> Phase calibration using target models (doPhaseCal)")
if not uvc:
# Get calibrated/averaged data
Aname = VLBAAIPSName(project, session)
uvname = project + "_" + session + "_" + band + "_Cal"
uvc = UV.newPAUV(uvname, Aname, parms["avgClass"], disk, seq, True,
err)
if err.isErr:
OErr.printErrMsg(err, "Error creating cal/avg AIPS data")
parms["targetModel"] = VLBAImageModel(parms["targets"],
parms["outTclass"], disk, seq,
err)
plotFile = project + "_" + session + "_" + band + ".PhaseCal.ps"
retCode = VLBAPhaseCal(uvc, err, calSou=parms["targets"], CalModel=parms["targetModel"], \
doCalib=-1, flagVer=0, doBand=-1, \
refAnt=goodCal["bestRef"], solInt=parms["manPCsolInt"], \
doSNPlot=parms["doSNPlot"], plotFile=plotFile, \
nThreads=nThreads, noScrat=noScrat, logfile=logFile, check=check, debug=debug)
if retCode != 0:
raise RuntimeError("Error in phase calibration")
VLBASaveOutFiles() # Save plot file in Outfiles
# Rewrite targets pickle file because targets may have been updated
SaveObject(parms["targets"], targetsPicklefile, True)
if len(parms["targets"]) <= 0:
logger.error(
"No target sources have been detected! Stopping pipeline.")
raise RuntimeError("No target sources have been detected!")
# Instrumental polarization calibration
if parms["doInstPol"]:
logger.info("--> Instrumental polarization calibration (doInstPol)")
# calibrators defaults to strong calibrator list
if not parms["instPolCal"]:
instPolCal = contCals
else:
instPolCal = parms["instPolCal"]
if not uvc:
# Get calibrated/averaged data
Aname = VLBAAIPSName(project, session)
uvname = project + "_" + session + "_" + band + "_Cal"
uvc = UV.newPAUV(uvname, Aname, parms["avgClass"], disk, seq, True,
err)
if err.isErr:
OErr.printErrMsg(err, "Error creating cal/avg AIPS data")
retCode = VLBAPolCal(uvc, instPolCal, err, \
doCalib=2, flagVer=0, doBand=-1, doSetJy=True, \
refAnt=goodCal["bestRef"], solInt=2.0, \
noScrat=noScrat, logfile=logFile, check=check, debug=debug)
if retCode != 0:
raise RuntimeError("Error in instrumental poln calibration")
# RL Phase (EVPA) calibration as BP table
if parms["doRLCal"] and parms["RLCal"]:
logger.info("--> RL phase calibration (doRLCal)")
if not uvc:
# Get calibrated/averaged data
Aname = VLBAAIPSName(project, session)
uvname = project + "_" + session + "_" + band + "_Cal"
uvc = UV.newPAUV(uvname, Aname, parms["avgClass"], disk, seq, True,
err)
if err.isErr:
OErr.printErrMsg(err, "Error creating cal/avg AIPS data")
retCode = VLBARLCal2(uvc, err, RLPCal=parms["RLCal"], \
doCalib=2, gainUse=2, flagVer=0, doBand=-1, doPol=True, \
refAnt=goodCal["bestRef"], niter=300, FOV=0.02/3600.0, \
nThreads=nThreads, noScrat=noScrat, logfile=logFile, check=check, debug=debug)
if retCode != 0:
raise RuntimeError("Error in RL phase calibration")
# image targets possible with Stokes I(QU)
if parms["doImgFullTarget"]:
logger.info("--> Image targets (doImgFullTargets)")
if not uvc:
# Get calibrated/averaged data
Aname = VLBAAIPSName(project, session)
uvname = project + "_" + session + "_" + band + "_Cal"
uvc = UV.newPAUV(uvname, Aname, parms["avgClass"], disk, seq, True,
err)
if err.isErr:
OErr.printErrMsg(err, "Error creating cal/avg AIPS data")
VLBAImageTargets(uvc,
err,
Sources=parms["targets"],
seq=seq,
sclass=parms["outIclass"],
doCalib=2,
flagVer=0,
doBand=-1,
Stokes=parms["Stokes"],
FOV=parms["FOV"],
Robust=parms["Robust"],
maxPSCLoop=2,
minFluxPSC=parms["minFluxPSC"],
solPInt=parms["solPInt"],
solMode="P",
maxASCLoop=parms["maxASCLoop"],
minFluxASC=parms["minFluxASC"],
solAInt=parms["solAInt"],
avgPol=parms["avgPol"],
avgIF=parms["avgIF"],
minSNR=parms["minSNR"],
refAnt=goodCal["bestRef"],
nTaper=parms["nTaper"],
Tapers=parms["Tapers"],
do3D=parms["do3D"],
nThreads=nThreads,
noScrat=noScrat,
logfile=logFile,
check=check,
debug=debug)
# Save UV data?
if parms["doSaveUV"] and (not check):
logger.info("--> Save UV data (doSaveUV)")
mess = "Write calibrated and averaged UV data to disk"
printMess(mess, logFile)
# Get calibrated/averaged data
if not uvc:
Aname = VLBAAIPSName(project, session)
uvname = project + "_" + session + "_" + band + "_Cal"
uvc = UV.newPAUV(uvname, Aname, parms["avgClass"], disk, seq, True,
err)
if err.isErr:
OErr.printErrMsg(err, "Error creating cal/avg AIPS data")
# Write
filename = project + session + band + "CalAvg.uvtab"
fuv = VLBAUVFITS(uvc, filename, 0, err, compress=Compress)
VLBAAddOutFile(filename, 'project', "Calibrated Averaged UV data")
# Save list of output files
VLBASaveOutFiles()
# Save UV data tables?
if parms["doSaveTab"] and (not check):
logger.info("--> Save UV data tables (doSaveTab)")
mess = "Write UV data tables to disk."
printMess(mess, logFile)
filename = project + session + band + "CalTab.uvtab"
fuv = VLBAUVFITSTab(uv, filename, 0, err)
VLBAAddOutFile(filename, 'project', "Calibrated AIPS tables")
# Save list of output files
VLBASaveOutFiles()
# Imaging results
outDisk = 0
if parms["doSaveImg"]:
logger.info("--> Save images (doSaveImg)")
mess = "Write images to disk."
printMess(mess, logFile)
# How many Stokes images
nstok = len(parms["Stokes"])
# Targets
logger.debug("parms['targets'] = " + str(parms['targets']))
for target in parms["targets"]:
if not check:
# intermediate images
oclass = parms["outTclass"]
x = Image.newPAImage("out", target, oclass, disk, seq, True,
err)
if (not x.exist):
print(target, "image not found. Skipping.")
continue
outfile = project + session + band + target + "." + oclass + ".fits"
mess = "Write Intermediate target " + outfile + " on disk " + str(
outDisk)
VLBAAddOutFile(outfile, target, 'Intermediate target image')
printMess(mess, logFile)
xf = VLBAImFITS(x, outfile, outDisk, err, fract=0.1)
# Save list of output files
VLBASaveOutFiles()
# Statistics
zz = imstat(x, err, logfile=logFile)
del x, xf
# Final images
for istok in range(0, nstok):
oclass = parms["Stokes"][istok:istok +
1] + parms["outIclass"][1:]
x = Image.newPAImage("out", target, oclass, disk, seq,
True, err)
outfile = project + session + band + target + "." + oclass + ".fits"
logger.info("Write " + outfile + " on disk " +
str(outDisk))
xf = VLBAImFITS(x, outfile, outDisk, err, fract=0.1)
VLBAAddOutFile(outfile, target, 'Image')
logger.info("Writing file " + outfile)
# Statistics
zz = imstat(x, err, logfile=logFile)
del x, xf
# Save list of output files
VLBASaveOutFiles()
# Calibrators
for target in parms["contCals"]:
if not check:
oclass = parms["outCclass"]
x = Image.newPAImage("out", target, oclass, disk, seq, True,
err)
if (not x.exist):
print(target, "image not found. Skipping.")
continue
outfile = project + session + band + target + "." + oclass + ".fits"
mess = "Write Calibrator " + outfile + " on disk " + str(
outDisk)
printMess(mess, logFile)
xf = VLBAImFITS(x, outfile, outDisk, err, fract=0.1)
VLBAAddOutFile(outfile, target, 'Calibrator Image')
# Statistics
zz = imstat(x, err, logfile=logFile)
del x, xf
# Save list of output files
VLBASaveOutFiles()
# end writing images loop
# Contour plots
if parms["doKntrPlots"]:
logger.info("--> Contour plots (doKntrPlots)")
VLBAKntrPlots(err,
imName=parms["targets"],
project=project,
session=session,
band=band,
disk=disk,
debug=debug)
# Save list of output files
VLBASaveOutFiles()
elif debug:
print("Not creating contour plots ( doKntrPlots = ",
parms["doKntrPlots"], " )")
# Source uv plane diagnostic plots
if parms["doDiagPlots"]:
logger.info("--> Diagnostic plots (doDiagPlots)")
# Get the highest number avgClass catalog file
Aname = VLBAAIPSName(project, session)
uvc = None
if not check:
uvname = project + "_" + session + "_" + band + "_Cal"
uvc = UV.newPAUV(uvname, Aname, parms["avgClass"], disk, seq, True,
err)
VLBADiagPlots( uvc, err, cleanUp=parms["doCleanup"], \
project=project, session=session, band=band, \
logfile=logFile, check=check, debug=debug )
# Save list of output files
VLBASaveOutFiles()
elif debug:
print("Not creating diagnostic plots ( doDiagPlots = ",
parms["doDiagPlots"], " )")
# Save metadata
srcMetadata = None
projMetadata = None
if parms["doMetadata"]:
logger.info("--> Save metadata (doMetadata)")
if not uvc:
# Get calibrated/averaged data
Aname = VLBAAIPSName(project, session)
uvname = project + "_" + session + "_" + band + "_Cal"
uvc = UV.newPAUV(uvname, Aname, parms["avgClass"], disk, seq, True,
err)
if err.isErr:
OErr.printErrMsg(err, "Error creating cal/avg AIPS data")
# Get source metadata; save to pickle file
srcMetadata = VLBASrcMetadata(uvc,
err,
Sources=parms["targets"],
seq=seq,
sclass=parms["outIclass"],
Stokes=parms["Stokes"],
logfile=logFile,
check=check,
debug=debug)
picklefile = project + "_" + session + "_" + band + "SrcReport.pickle"
SaveObject(srcMetadata, picklefile, True)
VLBAAddOutFile(picklefile, 'project', 'All source metadata')
# Get project metadata; save to pickle file
projMetadata = VLBAProjMetadata(uvc,
AIPS_VERSION,
err,
contCals=parms["contCals"],
goodCal=goodCal,
project=project,
session=session,
band=band,
dataInUVF=dataInUVF,
archFileID=archFileID)
picklefile = project + "_" + session + "_" + band + "ProjReport.pickle"
SaveObject(projMetadata, picklefile, True)
VLBAAddOutFile(picklefile, 'project', 'Project metadata')
# Write report
if parms["doHTML"]:
logger.info("--> Write HTML report (doHTML)")
VLBAHTMLReport( projMetadata, srcMetadata, \
outfile=project+"_"+session+"_"+band+"report.html", \
logFile=logFile )
# Write VOTable
if parms["doVOTable"]:
logger.info("--> Write VOTable (doVOTable)")
VLBAAddOutFile('VOTable.xml', 'project', 'VOTable report')
VLBAWriteVOTable(projMetadata, srcMetadata, filename='VOTable.xml')
# Save list of output files
VLBASaveOutFiles()
# Cleanup - delete AIPS files
if parms["doCleanup"] and (not check):
logger.info("--> Clean up (doCleanup)")
# Delete target images
# How many Stokes images
nstok = len(parms["Stokes"])
for istok in range(0, nstok):
oclass = parms["Stokes"][istok:istok + 1] + parms["outIclass"][1:]
AllDest(err, disk=disk, Aseq=seq, Aclass=oclass)
# delete Calibrator images
AllDest(err, disk=disk, Aseq=seq, Aclass=parms["outCclass"])
# Delete intermediate target images
AllDest(err, disk=disk, Aseq=seq, Aclass=parms["outTclass"])
OErr.printErrMsg(err, "Deleting AIPS images")
# Delete UV data
uv.Zap(err)
# Zap calibrated/averaged data
if not uvc:
Aname = VLBAAIPSName(project, session)
uvc = UV.newPAUV("AIPS UV DATA", Aname, parms["avgClass"], disk,
seq, True, err)
if err.isErr:
OErr.printErrMsg(err, "Error creating cal/avg AIPS data")
uvc.Zap(err)
OErr.printErrMsg(err, "Writing output/cleanup")
# Shutdown
mess = "Finished project " + project
printMess(mess, logFile)
OErr.printErr(err)
OSystem.Shutdown(ObitSys)
# Init Obit
err=OErr.OErr()
user = 101
ObitSys=OSystem.OSystem ("debug", 1, user, len(adirs), adirs, len(fdirs), fdirs, True, False, err)
OErr.printErrMsg(err, "Error with Obit startup")
# This shit really bites
AIPS.AIPS.userno = user
disk = 0
for ad in adirs:
disk += 1
AIPS.AIPS.disks.append(AIPS.AIPSDisk(None, disk, ad))
disk = 0
for fd in fdirs:
disk += 1
FITS.FITS.disks.append(FITS.FITSDisk(None, disk, ad))
# Set uv data
u3=UV.newPAUV("in","20051116", "K BAND", 3,1,True,err)
u3.Header(err)
VLACal.VLAClearCal(u3,err)
calModel="3C286_K.MODEL"
target="M87"
ACal="1331+305"
PCal="1239+075"
VLACal.VLACal(u3, target, ACal, err, calModel=calModel,calDisk=1)
VLACal.VLASplit(u3, target, err, outClass="IKDarr")
else:
slist = parms["targets"]
Report = ALMAReportTargets(uv, err, Sources=slist, seq=parms["seq"], sclass=outIClass, \
Stokes=parms["Stokes"], logfile=logFile, check=check, debug=debug)
# Save to pickle jar
ReportPicklefile = "./" + fileRoot + "Report.pickle" # Where results saved
SaveObject(Report, ReportPicklefile, True)
# Write results, cleanup
# Save cal/average UV data?
if parms["doSaveUV"] and (not check):
Aname = ALMAAIPSName(project, session)
cno = AIPSDir.PTestCNO(disk, user, Aname, avgClass[0:6], "UV",
parms["seq"], err)
if cno > 0:
uvt = UV.newPAUV("AIPS CAL UV DATA", Aname, avgClass, disk,
parms["seq"], True, err)
filename = parms["project"] + parms["session"] + parms[
"band"] + "Cal.uvtab"
fuv = ALMAUVFITS(uvt,
filename,
0,
err,
compress=parms["Compress"],
logfile=logFile)
ALMAAddOutFile(filename, 'project', "Calibrated Averaged UV data")
# Save list of output files
ALMASaveOutFiles()
del uvt
# Save raw UV data tables?
if parms["doSaveTab"] and (not check):
Aname = ALMAAIPSName(project, session)
if uv == None:
raise RuntimeError, "Cannot load " + inFile
# Load Data from Archive directory
if doLoadArchive:
uv = EVLAUVLoadArch(archRoot, project+session, loadClass, disk, seq, err, \
selConfig=selConfig, doSwPwr=doSwPwr, \
selBand=selBand, selChan=selChan, selNIF=selNIF, calInt=calInt, \
logfile=logFile, Compress=Compress, check=check, debug=debug)
if uv == None and not check:
raise RuntimeError, "Cannot load " + inFile
# Hanning
if doHann:
# Set uv if not done
if uv == None and not check:
uv = UV.newPAUV("AIPS UV DATA", project + session, loadClass, disk,
seq, True, err)
if err.isErr:
OErr.printErrMsg(err, "Error creating AIPS data")
uv = EVLAHann(uv, project+session, dataClass, disk, seq, err, \
logfile=logFile, check=check, debug=debug)
if uv == None and not check:
raise RuntimeError, "Cannot Hann data "
# Set uv is not done
if uv == None and not check:
uv = UV.newPAUV("AIPS UV DATA", project + session, dataClass, disk, seq,
True, err)
if err.isErr:
OErr.printErrMsg(err, "Error creating AIPS data")
inSeq = 2
inClass = 'UVCOP'
inSeq = 1
inClass = 'UVMOD'
inSeq = 1
outName = "Pytho3C286"
outName = "PythonTest" # Bigfield
outClass = 'ID'
outDisk = 1
outSeq = 3 # Mosaic
outSeq = 4 # Robust
outSeq = 2000 # Single
# Set data
print "Set data"
inData = UV.newPAUV("Input data", inName, inClass, inDisk, inSeq, True, err)
OErr.printErrMsg(err, "Error initializing")
# Make scratch version
print "Make scratch file"
scrData = UV.PScratch(inData, err)
OErr.printErrMsg(err, "Error making scratch file")
# Calibration/selection parameters
Input = UVImager.UVCalInput
Input['InData'] = inData
Input['OutData'] = scrData
Input['doCalSelect'] = True
Input['Stokes'] = 'I'
Input['BChan'] = 0
Input['EChan'] = 0
def VLAPolCal(uv, InsCals, RLCal, RLPhase, err, RM=0.0, \
doCalib=2, gainUse=0, flagVer=-1, \
soltype="APPR", fixPoln=False, avgIF=False, \
solInt=0.0, refAnt=0, \
pmodel=[0.0,0.0,0.0,0.0,0.0,0.0,0.0], \
FOV=0.05, niter = 100, \
nThreads=1, noScrat=[], logfile = ""):
""" Polarization calibration, both instrumental and orientation
Do Instrumental and R-L calibration
Instrumental cal uses PCAL, R-L cal is done by imaging each IF in Q and U
and summing the CLEAN components.
uv = UV data object to calibrate
InsCals = Instrumental poln calibrators, name or list of names
If None no instrumental cal
RLCal = R-L (polarization angle) calibrator,
If None no R-L cal
RLPhase = R-L phase of RLCal (deg) at 1 GHz
err = Obit error/message stack
RM = Rotation measure of RLCal
doCalib = Apply prior calibration table, positive=>calibrate
gainUse = CL/SN table to apply
flagVer = Input Flagging table version
soltype = solution type
fixPoln = if True, don't solve for source polarization in ins. cal
avgIF = if True, average IFs in ins. cal.
solInt = instrumental solution interval (min), 0=> scan average
refAnt = Reference antenna
pmodel = Instrumental poln cal source poln model.
pmodel[0] = I flux density (Jy)
pmodel[1] = Q flux density (Jy)
pmodel[2] = U flux density (Jy)
pmodel[3] = V flux density (Jy)
pmodel[4] = X offset in sky (arcsec)
pmodel[5] = Y offset in sky (arcsec)
FOV = field of view radius (deg) needed to image RLCal
niter = Number of iterations of CLEAN in R-L cal
nThreads = Number of threads to use in imaging
noScrat = list of disks to avoid for scratch files
logfile = Log file for task
"""
################################################################
# Instrumental calibrtation
if InsCals!=None:
pcal = AIPSTask.AIPSTask("pcal")
pcal.logFile = logfile
setname(uv,pcal)
if type(InsCals)==list:
pcal.calsour[1:] = InsCals
else:
pcal.calsour[1:] = [InsCals]
pcal.docalib = doCalib
pcal.gainuse = gainUse
pcal.flagver = flagVer
pcal.soltype = soltype
pcal.solint = solInt
pcal.refant = refAnt
if fixPoln:
pcal.bparm[10]=1.0
if avgIF:
pcal.cparm[1]=1.0
pcal.pmodel[1:] = pmodel
pcal.i
pcal.g
# end instrumental poln cal
# R-L phase cal
if RLCal!=None:
img = ObitTask.ObitTask("Imager")
img.logFile = logfile
setname(uv,img)
img.doCalib = doCalib
img.gainUse = gainUse
img.flagVer = flagVer
img.doPol = True
img.Sources[0] = RLCal
img.Stokes = "IQU"
img.FOV = FOV
img.Niter = niter
img.autoWindow = True
img.dispURL = "None"
img.Catalog = "None"
img.nThreads = nThreads
img.noScrat = noScrat
# Temporary output files
if img.DataType=="AIPS":
img.outName = "TEMP"
img.outClass= "IPOLCL"
img.outDisk = img.inDisk
img.outSeq = 6666
img.out2Name = "TEMP"
img.out2Class= "IPOLCL"
img.out2Disk = img.inDisk
img.out2Seq = 7777
elif img.DataType=="FITS":
img.outFile = "TEMPPOLCAL.fits"
img.outDisk = img.inDisk
img.out2File = "TEMPPOLCAL2.uvtab"
img.out2Disk = img.inDisk
# How many IFs?
h = uv.Desc.Dict
if h["jlocif"]>=0:
nif = h["inaxes"][h["jlocif"]]
else:
nif = 1
# Lists of flux densities and RMSes
IFlux = []
IRMS = []
QFlux = []
QRMS = []
UFlux = []
URMS = []
# Loop over IF imaging I,Q, U
for iif in range (1, nif+1):
img.BIF = iif
img.EIF = iif
#img.dispURL = "ObitView" # DEBUG
#img.debug=True # DEBUG
img.g
# Get fluxes from inner quarter of images
if img.DataType=="AIPS":
outName = (img.Sources[0].strip()+"TEMP")[0:12]
outDisk = img.outDisk
outSeq = 6666
# Stokes I
outClass="IPOLCL"
x = Image.newPAImage("I",outName, outClass, outDisk,outSeq,True,err)
h = x.Desc.Dict
blc = [h["inaxes"][0]/4,h["inaxes"][1]/4]
trc = [3*h["inaxes"][0]/4,3*h["inaxes"][1]/4]
stat = imstat(x, err, blc=blc,trc=trc)
IFlux.append(stat["Flux"])
IRMS.append(stat["RMSHist"])
x.Zap(err) # Cleanup
del x
# Stokes Q
outClass="QPOLCL"
x = Image.newPAImage("Q",outName, outClass, outDisk,outSeq,True,err)
stat = imstat(x, err, blc=blc,trc=trc)
QFlux.append(stat["Flux"])
QRMS.append(stat["RMSHist"])
x.Zap(err) # Cleanup
del x
# Stokes U
outClass="UPOLCL"
x = Image.newPAImage("U",outName, outClass, outDisk,outSeq,True,err)
stat = imstat(x, err, blc=blc,trc=trc)
UFlux.append(stat["Flux"])
URMS.append(stat["RMSHist"])
x.Zap(err) # Cleanup
del x
# Delete UV output
out2Name = (img.Sources[0].strip()+"TEMP")[0:12]
out2Class="IPOLCL"
out2Disk = img.inDisk
out2Seq = 7777
u = UV.newPAUV("UV",out2Name,out2Class,out2Disk,out2Seq,True,err)
u.Zap(err)
del u
elif img.DataType=="FITS":
# Stokes I
outFile = img.Sources[0].strip()+"ITEMPPOLCAL.fits"
x = Image.newPFImage("I",outFile,img.outDisk,True,err)
h = x.Desc.Dict
blc = [h["inaxes"][0]/4,h["inaxes"][1]/4]
trc = [3*h["inaxes"][0]/4,3*h["inaxes"][1]/4]
stat = imstat(x, err, blc=blc,trc=trc)
IFlux.append(stat["Flux"])
IRMS.append(stat["RMSHist"])
x.Zap(err) # Cleanup
del x
# Stokes Q
outFile = img.Sources[0].strip()+"ITEMPPOLCAL.fits"
x = Image.newPFImage("Q",outFile,img.outDisk,True,err)
stat = imstat(x, err, blc=blc,trc=trc)
IFlux.append(stat["Flux"])
IRMS.append(stat["RMSHist"])
x.Zap(err) # Cleanup
del x
# Stokes U
outFile = img.Sources[0].strip()+"ITEMPPOLCAL.fits"
x = Image.newPFImage("Q",outFile,img.outDisk,True,err)
stat = imstat(x, err, blc=blc,trc=trc)
IFlux.append(stat["Flux"])
IRMS.append(stat["RMSHist"])
x.Zap(err) # Cleanup
del x
out2File = img.Sources[0].strip()+"TEMPPOLCAL2.uvtab"
u = UV.newPFUV("UV",outFile,img.outDisk,True,err)
u.Zap(err)
del u
# End accumulate statistics by file type
# End loop over IF
# Give results, compute R-L correction
RLCor = []
import math
mess = " IF IFlux IRMS QFlux QRMS UFlux URMS R-L Corr"
printMess(mess, logfile)
for i in range (0,len(IFlux)):
# REALLY NEED RM Correction!!!!!
cor = RLPhase - 57.296 * math.atan2(UFlux[i],QFlux[i])
RLCor.append(cor)
mess = "%3d %8.3f %8.3f %7.3f %7.3f %7.3f %7.3f %7.3f "%\
(i+1, IFlux[i], IRMS[i], QFlux[i], QRMS[i], UFlux[i], URMS[i], cor)
printMess(mess, logfile)
# Copy highest CL table
hiCL = uv.GetHighVer("AIPS CL")
# Apply R-L phase corrections
clcor = AIPSTask.AIPSTask("clcor")
clcor.logFile = logfile
setname(uv,clcor)
clcor.opcode = "POLR"
clcor.gainver = hiCL
clcor.gainuse = hiCL+1
clcor.clcorprm[1:] = RLCor
clcor.g
pass
setname(uv, blavg)
blavg.Sources = targets.split(',')
#blavg.Stokes=' '
blavg.FOV = 1.3
blavg.maxInt = 1.0
blavg.maxFact = 1.01
blavg.outDType = 'AIPS'
blavg.outName = 'IMAGE'
blavg.outClass = 'BLAVG'
blavg.outSeq = 1
blavg.outDisk = 1
blavg.taskLog = 'IMAGE.log'
blavg.g
uv = UV.newPAUV('BL IMAGE DATA', 'IMAGE', 'BLAVG', 1, 1, True, err)
#uv=OTObit.uvlod(args[0],0,nam,cls,disk,seq,err)
####### Initialize parameters dictionary #####
#parms = KATInitContParms()
#Now we should have a uv file in our aips disk
KATImageTargets (uv, err, Sources=targets, seq=1, sclass="IClean", OutlierArea=parms["outlierArea"],\
doCalib=-1, doBand=-1, flagVer=-1, doPol=parms["doPol"], PDVer=parms["PDVer"], \
Stokes=parms["Stokes"], FOV=parms["FOV"], Robust=parms["Robust"], Niter=parms["Niter"], \
CleanRad=parms["CleanRad"], minFlux=parms["minFlux"], OutlierSize=parms["OutlierSize"], \
xCells=parms["xCells"], yCells=parms["yCells"], Reuse=parms["Reuse"], minPatch=parms["minPatch"], \
maxPSCLoop=parms["maxPSCLoop"], minFluxPSC=parms["minFluxPSC"], noNeg=parms["noNeg"], \
solPInt=parms["solPInt"], solPMode=parms["solPMode"], solPType=parms["solPType"], \
maxASCLoop=parms["maxASCLoop"], minFluxASC=parms["minFluxASC"], nx=parms["nx"], ny=parms["ny"], \
def SubPeel(uv, source, imp, uvp, err, addBack=False, seq=999, \
flagVer=0, nThreads=1, doGPU=False, noScrat=[0,0,0], taskLog='', debug=False):
"""
Subtract Peel model w/ solutions, then optionally add back w/o corruptions
UV data should have calibration tables from self calibration
Output data will be on the same disk as the input, seq=seq, class='PelSub' and
with name = source (up to 12 char).
Returns Peel source subtracted/replaced data
* uv Dataset with cal tables
Needs at least the self cal gain table
* source source name
* imp Peel source model (CC table from ImagePeel)
* uvp UV data the result of peel (ImagePeel)
* err Python Obit Error/message stack
* seq Sequence number for output
* addBack Add model back to data w/o corruptions? Not recommended.
* flagVer FG table to apply, -1=> no flag
* nThreads number of threads to use
* doGPU Use GPU if available?
* noScrat AIPS disks not to use for scratch
* taskLog Log file
* debug If True leave debug Input file in /tmp
"""
################################################################
# Checks
if not UV.PIsA(uv):
raise TypeError("uv MUST be a Python Obit UV data")
if not Image.PIsA(imp):
raise TypeError("imp MUST be a Python Obit Image")
if not OErr.OErrIsA(err):
raise TypeError("err MUST be an OErr")
# Split main data set
OErr.PLog(err, OErr.Info, "Copy data")
OErr.printErr(err)
split = ObitTask('Split')
setname(uv, split)
split.outDisk = split.inDisk
split.outSeq = seq
split.outClass = 'UPeel'
split.Sources[0] = source
split.flagVer = flagVer
if uv.GetHighVer('AIPS SN') > 0:
split.doCalib = 2
split.gainUse = 0
split.doCalWt = True
else:
split.doCalib = -1
split.taskLog = taskLog
split.debug = debug
split.g
outClass = split.outClass
outDisk = split.outDisk
outSeq = split.outSeq
# Get data
OErr.PLog(err, OErr.Info, "Make Peel model with corruptions")
OErr.printErr(err)
if UV.AExist(source[0:12], outClass, outDisk, outSeq, err):
datauv = UV.newPAUV('data', source[0:12], outClass, outDisk, outSeq,
True, err)
else:
datauv = UV.newPAUV('data', source[0:8], outClass, outDisk, outSeq,
True, err)
# Make data set with the model peel source with peel cal applied
uvsub = ObitTask('UVSub')
setname(uv, uvsub)
uvsub.outName = source[0:12]
uvsub.outDisk = uvsub.inDisk
uvsub.outSeq = 1
uvsub.outClass = 'Model'
uvsub.Sources[0] = source
uvsub.flagVer = flagVer
uvsub.doCalib = -1
uvsub.gainUse = 0
set2name(imp, uvsub)
uvsub.CCVer = 1
uvsub.nfield = 1
uvsub.Cmethod = 'DFT'
uvsub.Opcode = 'MODL'
uvsub.PBCor = False
uvsub.noScrat = noScrat
uvsub.noNeg = False
uvsub.taskLog = taskLog
uvsub.nThreads = nThreads
uvsub.doGPU = doGPU
uvsub.debug = debug
uvsub.g
# Get model data
modeluv = UV.newPAUV('model', uvsub.outName, uvsub.outClass, uvsub.outDisk,
uvsub.outSeq, True, err)
# Copy/invert/unblank SN table from peeluv
hiPeelSN = uvp.GetHighVer('AIPS SN')
inTab = max(1, hiPeelSN)
sntab = uvp.NewTable(Table.READONLY, 'AIPS SN', inTab, err)
z = UVSelfCal.PInvertSN(sntab, modeluv, 1, True, err)
# Apply calibration table in subtract
modeluv.List.set('doCalSelect', True)
modeluv.List.set('doCalib', 2)
modeluv.List.set('gainUse', 1)
modeluv.List.set('passAll', True)
# Subtract model from main data
OErr.PLog(err, OErr.Info, "Subtract Corrupted Peel model from uv data")
UV.PUtilVisSub(datauv, modeluv, datauv, err)
OErr.printErr(err)
# Add model without corrupting calibration
if addBack:
OErr.PLog(err, OErr.Info, "Add Peel model without corruptions")
uvsub = ObitTask('UVSub')
setname(datauv, uvsub)
setoname(datauv, uvsub)
uvsub.outSeq = uvsub.inSeq + 1
uvsub.Sources[0] = source
uvsub.flagVer = flagVer
uvsub.doCalib = -1
uvsub.gainUse = 0
set2name(imp, uvsub)
uvsub.CCVer = 1
uvsub.nfield = 1
uvsub.Cmethod = 'DFT'
uvsub.Factor = -1.
uvsub.PBCor = False
uvsub.noScrat = noScrat
uvsub.noNeg = False
uvsub.taskLog = taskLog
uvsub.nThreads = nThreads
uvsub.doGPU = doGPU
uvsub.debug = debug
uvsub.g
outClass = uvsub.outClass
outDisk = uvsub.outDisk
outSeq = uvsub.outSeq
# end add back
OErr.printErr(err)
# Delete model dataset
if not debug:
modeluv.Zap(err)
# final data
if UV.AExist(source[0:12], outClass, outDisk, outSeq, err):
datauv2 = UV.newPAUV('data', source[0:12], outClass, outDisk, outSeq,
True, err)
else:
datauv2 = UV.newPAUV('data', source[0:8], outClass, outDisk, outSeq,
True, err)
return datauv2
def MKContPipeline(files, outputdir, **kwargs):
"""MeerKAT Continuum pipeline.
Parameters
----------
files : list
h5 filenames (note: support for multiple h5 files
i.e. ConcatenatedDataSet is not currently supported)
outputdir : string
Directory location to write output data,
scratchdir : string, optional
The directory location of the aips disk
parmFile : string, optional
Overwrite the default imaging parameters using this parameter file.
"""
#if len(files) > 1:
# raise TooManyKatfilesException('Processing multiple katfiles are not currently supported')
h5file = files
# Die gracefully if we cannot write to the output area...
if not os.path.exists(outputdir):
print('Specified output directory: '+ outputdir + 'does not exist.')
exit(-1)
# Obit error logging
err = OErr.OErr()
#################### Initialize filenames #######################################################
fileRoot = os.path.join(outputdir, os.path.basename(os.path.splitext(files[0])[0])) # root of file name
logFile = fileRoot+".log" # Processing log file
avgClass = ("UVAv")[0:6] # Averaged data AIPS class
manifestfile = outputdir + '/manifest.pickle'
############################# Initialize OBIT and AIPS ##########################################
noScrat = []
# Logging directly to logFile
OErr.PInit(err, 2, logFile)
EVLAAddOutFile(os.path.basename(logFile), 'project', 'Pipeline log file')
ObitSys = AIPSSetup.AIPSSetup(err,configfile=kwargs.get('configFile'),scratchdir=kwargs.get('scratchdir'))
# Get the set up AIPS environment.
AIPS_ROOT = os.environ['AIPS_ROOT']
AIPS_VERSION = os.environ['AIPS_VERSION']
nThreads = 24
user = OSystem.PGetAIPSuser()
AIPS.userno = user
disk = 1
fitsdisk = 0
nam = os.path.basename(os.path.splitext(files[0])[0])[0:10]
cls = "Raw"
seq = 1
############################# Initialise Parameters ##########################################
####### Initialize parameters dictionary #####
parms = KATInitContParms()
####### User defined parameters ######
if kwargs.get('parmFile'):
print("parmFile",kwargs.get('parmFile'))
exec(open(kwargs.get('parmFile')).read())
EVLAAddOutFile(os.path.basename(kwargs.get('parmFile')), 'project', 'Pipeline input parameters' )
############### Initialize katfile object, uvfits object and condition data #########################
OK = False
# Open the h5 file as a katfile object
try:
#open katfile and perform selection according to kwargs
katdata = katfile.open(h5file)
OK = True
except Exception as exception:
print(exception)
if not OK:
OErr.PSet(err)
OErr.PLog(err, OErr.Fatal, "Unable to read KAT HDF5 data in " + str(h5file))
raise KATUnimageableError("Unable to read KAT HDF5 data in " + str(h5file))
#We have a katdal object- read some flags and ad them in in available
if kwargs.get('flags') is not None:
flags=kwargs.get('flags')
fa = flags.split(',')
for fn,ff in enumerate(fa):
ex_flags_file = h5py.File(ff)
ex_flags = da.from_array(ex_flags_file['flags'], chunks=(1,342,katdata.shape[2]))
#Sum the new flags
katdata.datasets[fn].source.data.flags = ex_flags
#Are we MeerKAT or KAT-7
telescope = katdata.ants[0].name[0]
if telescope=='m':
sefd=500.
else:
sefd=1200.
#Get calibrator models
fluxcals = katpoint.Catalogue(file(FITSDir.FITSdisks[0]+"/"+parms["fluxModel"]))
#Condition data (get bpcals, update names for aips conventions etc)
KATh5Condition(katdata,fluxcals,err)
###################### Data selection and static edits ############################################
# Select data based on static imageable parameters
MKATh5Select(katdata, parms, err, **kwargs)
####################### Import data into AIPS #####################################################
# Construct a template uvfits file from master template
mastertemplate=ObitTalkUtil.FITSDir.FITSdisks[fitsdisk]+'MKATTemplate.uvtab.gz'
outtemplate=nam+'.uvtemp'
KATH5toAIPS.MakeTemplate(mastertemplate,outtemplate,len(katdata.channel_freqs))
uv=OTObit.uvlod(outtemplate,0,nam,cls,disk,seq,err)
obsdata = KATH5toAIPS.KAT2AIPS(katdata, uv, disk, fitsdisk, err, calInt=1.0, **kwargs)
MakeIFs.UVMakeIF(uv,8,err)
# Print the uv data header to screen.
uv.Header(err)
os.remove(outtemplate)
############################# Set Project Processing parameters ###################################
# Parameters derived from obsdata and katdata
MKATGetObsParms(obsdata, katdata, parms, logFile)
###### Initialise target parameters #####
KATInitTargParms(katdata,parms,err)
# General AIPS data parameters at script level
dataClass = ("UVDa")[0:6] # AIPS class of raw uv data
band = katdata.spectral_windows[0].product #Correlator product
project = parms["project"][0:12] # Project name (12 char or less, used as AIPS Name)
outIClass = parms["outIClass"] # image AIPS class
debug = parms["debug"]
check = parms["check"]
# Load the outputs pickle jar
EVLAFetchOutFiles()
OSystem.PAllowThreads(nThreads) # Allow threads in Obit/oython
retCode = 0
maxgap = max(parms["CalAvgTime"],20*katdata.dump_period)/60.
################### Start processing ###############################################################
mess = "Start project "+parms["project"]+" AIPS user no. "+str(AIPS.userno)+\
", KAT7 configuration "+parms["KAT7Cfg"]
printMess(mess, logFile)
if debug:
pydoc.ttypager = pydoc.plainpager # don't page task input displays
mess = "Using Debug mode "
printMess(mess, logFile)
if check:
mess = "Only checking script"
printMess(mess, logFile)
# Log parameters
printMess("Parameter settings", logFile)
for p in parms:
mess = " "+p+": "+str(parms[p])
printMess(mess, logFile)
clist = []
for DCal in parms["DCals"]:
if DCal["Source"] not in clist:
clist.append(DCal["Source"])
for PCal in parms["PCals"]:
if PCal["Source"] not in clist:
clist.append(PCal["Source"])
for ACal in parms["ACals"]:
if ACal["Source"] not in clist:
clist.append(ACal["Source"])
if kwargs.get('targets') is not None:
targets = [targ.name for targ in katdata.catalogue if (targ.name not in clist) and (targ.name in kwargs.get('targets').split(','))]
else:
targets = [targ.name for targ in katdata.catalogue if (targ.name not in clist)]
refAnt = FetchObject(fileRoot+".refAnt.pickle")
# Save parameters to pickle jar, manifest
ParmsPicklefile = fileRoot+".Parms.pickle" # Where results saved
SaveObject(parms, ParmsPicklefile, True)
EVLAAddOutFile(os.path.basename(ParmsPicklefile), 'project', 'Processing parameters used' )
loadClass = dataClass
retCode = KATCalAvg (uv, "PREAVG", parms["seq"], parms["CalAvgTime"], err, \
flagVer=-1, doCalib=-1, gainUse=-1, doBand=-1, BPVer=-1, doPol=False, \
avgFreq=0, chAvg=1, BChan=1, EChan=0, doAuto=parms["doAuto"], Stokes=' ',\
BIF=parms["CABIF"], EIF=parms["CAEIF"], Compress=parms["Compress"], \
nThreads=nThreads, logfile=logFile, check=check, debug=debug)
if retCode!=0:
raise RuntimeError("Error in CalAvg")
uv.Zap(err)
# Get initially averaged data
if not check:
uv = UV.newPAUV("AIPS UV DATA", EVLAAIPSName(project), "PREAVG", \
disk, parms["seq"], True, err)
if err.isErr:
OErr.printErrMsg(err, "Error creating initial avg AIPS data")
# Hanning
parms["doHann"]=True
if parms["doHann"]:
# Set uv if not done
if uv==None and not check:
uv = UV.newPAUV("AIPS UV DATA", EVLAAIPSName(project), loadClass[0:6], disk, parms["seq"], True, err)
if err.isErr:
OErr.printErrMsg(err, "Error creating AIPS data")
uv = KATHann(uv, EVLAAIPSName(project), dataClass, disk, parms["seq"], err, \
doDescm=parms["doDescm"], flagVer=0, logfile=logFile, check=check, debug=debug)
#Halve channels after hanning.
parms["selChan"]=int(parms["selChan"]/2)
parms["BChDrop"]=int(parms["BChDrop"]/2)
parms["EChDrop"]=int(parms["EChDrop"]/2)
if uv==None and not check:
raise RuntimeError("Cannot Hann data ")
# Clear any old calibration/editing
if parms["doClearTab"]:
mess = "Clear previous calibration"
printMess(mess, logFile)
EVLAClearCal(uv, err, doGain=parms["doClearGain"], doFlag=parms["doClearFlag"], doBP=parms["doClearBP"], check=check)
OErr.printErrMsg(err, "Error resetting calibration")
# Quack to remove data from start and end of each scan
if parms["doQuack"]:
retCode = EVLAQuack (uv, err, begDrop=parms["quackBegDrop"], endDrop=parms["quackEndDrop"], \
Reason=parms["quackReason"], \
logfile=logFile, check=check, debug=debug)
if retCode!=0:
raise RuntimeError("Error Quacking data")
# Flag antennas shadowed by others?
if parms["doShad"]:
retCode = EVLAShadow (uv, err, shadBl=parms["shadBl"], \
logfile=logFile, check=check, debug=debug)
if retCode!=0:
raise RuntimeError("Error Shadow flagging data")
# Median window time editing, for RFI impulsive in time
if parms["doMednTD1"]:
mess = "Median window time editing, for RFI impulsive in time:"
printMess(mess, logFile)
retCode = EVLAMedianFlag (uv, clist, err, noScrat=noScrat, nThreads=nThreads, \
avgTime=parms["mednAvgTime"], avgFreq=parms["mednAvgFreq"], chAvg= parms["mednChAvg"], \
timeWind=parms["mednTimeWind"],flagVer=2, flagTab=2,flagSig=parms["mednSigma"], \
logfile=logFile, check=check, debug=False)
if retCode!=0:
raise RuntimeError("Error in MednFlag")
# Median window frequency editing, for RFI impulsive in frequency
if parms["doFD1"]:
mess = "Median window frequency editing, for RFI impulsive in frequency:"
printMess(mess, logFile)
retCode = EVLAAutoFlag (uv, clist, err, flagVer=2, flagTab=2, doCalib=-1, doBand=-1, \
timeAvg=parms["FD1TimeAvg"], \
doFD=True, FDmaxAmp=1.0e20, FDmaxV=1.0e20, FDwidMW=parms["FD1widMW"], \
FDmaxRMS=[1.0e20,0.1], FDmaxRes=parms["FD1maxRes"], \
FDmaxResBL= parms["FD1maxRes"], FDbaseSel=parms["FD1baseSel"],\
nThreads=nThreads, logfile=logFile, check=check, debug=debug)
if retCode!=0:
raise RuntimeError("Error in AutoFlag")
# Parallactic angle correction?
if parms["doPACor"]:
retCode = EVLAPACor(uv, err, \
logfile=logFile, check=check, debug=debug)
if retCode!=0:
raise RuntimeError("Error in Parallactic angle correction")
# Need to find a reference antenna? See if we have saved it?
if (parms["refAnt"]<=0):
refAnt = FetchObject(fileRoot+".refAnt.pickle")
if refAnt:
parms["refAnt"] = refAnt
# Use bandpass calibrator and center half of each spectrum
if parms["refAnt"]<=0:
mess = "Find best reference antenna: run Calib on BP Cal(s) "
printMess(mess, logFile)
parms["refAnt"] = EVLAGetRefAnt(uv, parms["BPCals"], err, flagVer=0, \
solInt=parms["bpsolint1"], nThreads=nThreads, \
logfile=logFile, check=check, debug=debug)
if err.isErr:
raise RuntimeError("Error finding reference antenna")
if parms["refAnts"][0]<=0:
parms["refAnts"][0] = parms["refAnt"]
mess = "Picked reference antenna "+str(parms["refAnt"])
printMess(mess, logFile)
# Save it
ParmsPicklefile = fileRoot+".Parms.pickle" # Where results saved
SaveObject(parms, ParmsPicklefile, True)
refAntPicklefile = fileRoot+".refAnt.pickle" # Where results saved
SaveObject(parms["refAnt"], refAntPicklefile, True)
# Plot Raw, edited data?
parms["doRawSpecPlot"]=False
parms["doSpecPlot"]=False
if parms["doRawSpecPlot"] and parms["plotSource"]:
mess = "Raw Spectral plot for: "+' '.join(parms["BPCal"])
printMess(mess, logFile)
plotFile = fileRoot+"_RawSpec.ps"
retCode = EVLASpectrum(uv, parms["BPCal"], parms["plotTime"], maxgap, plotFile, parms["refAnt"], err, \
Stokes=["RR","LL"], doband=-1, \
check=check, debug=debug, logfile=logFile )
if retCode!=0:
raise RuntimeError("Error in Plotting spectrum")
EVLAAddOutFile(plotFile, 'project', 'Pipeline log file' )
# delay calibration
if parms["doDelayCal"] and parms["DCals"] and not check:
plotFile = fileRoot+"_DelayCal.ps"
retCode = EVLADelayCal(uv, parms["DCals"], err, \
BChan=parms["delayBChan"], EChan=parms["delayEChan"], \
doCalib=2, flagVer=0, doBand=-1, \
solInt=parms["delaySolInt"], smoTime=parms["delaySmoo"], \
refAnts=[parms["refAnt"]], doTwo=parms["doTwo"],
doZeroPhs=parms["delayZeroPhs"], \
doPlot=parms["doSNPlot"], plotFile=plotFile, \
nThreads=nThreads, noScrat=noScrat, \
logfile=logFile, check=check, debug=debug)
if retCode!=0:
raise RuntimeError("Error in delay calibration")
# Plot corrected data?
if parms["doSpecPlot"] and parms["plotSource"]:
plotFile = fileRoot+"_DelaySpec.ps"
retCode = EVLASpectrum(uv, parms["BPCal"], parms["plotTime"], maxgap, \
plotFile, parms["refAnt"], err, \
Stokes=["RR","LL"], doband=-1, \
check=check, debug=debug, logfile=logFile )
if retCode!=0:
raise RuntimeError("Error in Plotting spectrum")
print(parms["bpBChan1"],parms["bpEChan1"],parms["bpBChan2"],parms["bpEChan2"],parms["bpChWid2"])
# Bandpass calibration
if parms["doBPCal"] and parms["BPCals"]:
retCode = KATBPCal(uv, parms["BPCals"], err, noScrat=noScrat, solInt1=parms["bpsolint1"], \
solInt2=parms["bpsolint2"], solMode=parms["bpsolMode"], \
BChan1=parms["bpBChan1"], EChan1=parms["bpEChan1"], \
BChan2=parms["bpBChan2"], EChan2=parms["bpEChan2"], ChWid2=parms["bpChWid2"], \
doCenter1=parms["bpDoCenter1"], refAnt=parms["refAnt"], \
UVRange=parms["bpUVRange"], doCalib=2, gainUse=0, flagVer=0, doPlot=False, \
nThreads=nThreads, logfile=logFile, check=check, debug=debug)
if retCode!=0:
raise RuntimeError("Error in Bandpass calibration")
# Plot corrected data?
if parms["doSpecPlot"] and parms["plotSource"]:
plotFile = fileRoot+"_BPSpec.ps"
retCode = EVLASpectrum(uv, parms["BPCal"], parms["plotTime"], maxgap, plotFile, \
parms["refAnt"], err, Stokes=["RR","LL"], doband=1, \
check=check, debug=debug, logfile=logFile )
if retCode!=0:
raise RuntimeError("Error in Plotting spectrum")
# Amp & phase Calibrate
if parms["doAmpPhaseCal"]:
plotFile = fileRoot+"_APCal.ps"
retCode = KATCalAP (uv, [], parms["ACals"], err, PCals=parms["PCals"],
doCalib=2, doBand=1, BPVer=1, flagVer=0, \
BChan=parms["ampBChan"], EChan=parms["ampEChan"], \
solInt=parms["solInt"], solSmo=parms["solSmo"], ampScalar=parms["ampScalar"], \
doAmpEdit=parms["doAmpEdit"], ampSigma=parms["ampSigma"], \
ampEditFG=parms["ampEditFG"], \
doPlot=parms["doSNPlot"], plotFile=plotFile, refAnt=parms["refAnt"], \
nThreads=nThreads, noScrat=noScrat, logfile=logFile, check=check, debug=debug)
#print parms["ACals"],parms["PCals"]
if retCode!=0:
raise RuntimeError("Error calibrating")
# More editing
if parms["doAutoFlag"]:
mess = "Post calibration editing:"
printMess(mess, logFile)
# if going to redo then only calibrators
if parms["doRecal"]:
# Only calibrators
clist = []
for DCal in parms["DCals"]:
if DCal["Source"] not in clist:
clist.append(DCal["Source"])
for PCal in parms["PCals"]:
if PCal["Source"] not in clist:
clist.append(PCal["Source"])
for ACal in parms["ACals"]:
if ACal["Source"] not in clist:
clist.append(ACal["Source"])
else:
clist = []
retCode = EVLAAutoFlag (uv, clist, err, flagVer=0, flagTab =2, \
doCalib=2, gainUse=0, doBand=1, BPVer=1, \
IClip=parms["IClip"], minAmp=parms["minAmp"], timeAvg=parms["timeAvg"], \
doFD=parms["doAFFD"], FDmaxAmp=parms["FDmaxAmp"], FDmaxV=parms["FDmaxV"], \
FDwidMW=parms["FDwidMW"], FDmaxRMS=parms["FDmaxRMS"], \
FDmaxRes=parms["FDmaxRes"], FDmaxResBL=parms["FDmaxResBL"], \
FDbaseSel=parms["FDbaseSel"], \
nThreads=nThreads, logfile=logFile, check=check, debug=debug)
if retCode!=0:
raise RuntimeError("Error in AutoFlag")
# Redo the calibration using new flagging?
if parms["doBPCal2"]==None:
parms["doBPCal2"] = parms["doBPCal"]
if parms["doDelayCal2"]==None:
parms["doDelayCal2"] = parms["doDelayCal2"]
if parms["doAmpPhaseCal2"]==None:
parms["doAmpPhaseCal2"] = parms["doAmpPhaseCal"]
if parms["doAutoFlag2"]==None:
parms["doAutoFlagCal2"] = parms["doAutoFlag"]
if parms["doRecal"]:
mess = "Redo calibration:"
printMess(mess, logFile)
EVLAClearCal(uv, err, doGain=True, doFlag=False, doBP=True, check=check, logfile=logFile)
OErr.printErrMsg(err, "Error resetting calibration")
# Parallactic angle correction?
if parms["doPACor"]:
retCode = EVLAPACor(uv, err, \
logfile=logFile, check=check, debug=debug)
if retCode!=0:
raise RuntimeError("Error in Parallactic angle correction")
# Delay recalibration
if parms["doDelayCal2"] and parms["DCals"] and not check:
plotFile = fileRoot+"_DelayCal2.ps"
retCode = EVLADelayCal(uv, parms["DCals"], err, \
BChan=parms["delayBChan"], EChan=parms["delayEChan"], \
doCalib=2, flagVer=0, doBand=-1, \
solInt=parms["delaySolInt"], smoTime=parms["delaySmoo"], \
refAnts=[parms["refAnt"]], doTwo=parms["doTwo"], \
doZeroPhs=parms["delayZeroPhs"], \
doPlot=parms["doSNPlot"], plotFile=plotFile, \
nThreads=nThreads, noScrat=noScrat, \
logfile=logFile, check=check, debug=debug)
if retCode!=0:
raise RuntimeError("Error in delay calibration")
# Plot corrected data?
if parms["doSpecPlot"] and parms["plotSource"]:
plotFile = fileRoot+"_DelaySpec2.ps"
retCode = EVLASpectrum(uv, parms["BPCal"], parms["plotTime"], maxgap, plotFile, parms["refAnt"], err, \
Stokes=["RR","LL"], doband=-1, \
check=check, debug=debug, logfile=logFile )
if retCode!=0:
raise RuntimeError("Error in Plotting spectrum")
# Bandpass calibration
if parms["doBPCal2"] and parms["BPCals"]:
retCode = KATBPCal(uv, parms["BPCals"], err, noScrat=noScrat, solInt1=parms["bpsolint1"], \
solInt2=parms["bpsolint2"], solMode=parms["bpsolMode"], \
BChan1=parms["bpBChan1"], EChan1=parms["bpEChan1"], \
BChan2=parms["bpBChan2"], EChan2=parms["bpEChan2"], ChWid2=parms["bpChWid2"], \
doCenter1=parms["bpDoCenter1"], refAnt=parms["refAnt"], \
UVRange=parms["bpUVRange"], doCalib=2, gainUse=0, flagVer=0, doPlot=False, \
nThreads=nThreads, logfile=logFile, check=check, debug=debug)
if retCode!=0:
raise RuntimeError("Error in Bandpass calibration")
# Plot corrected data?
if parms["doSpecPlot"] and parms["plotSource"]:
plotFile = fileRoot+"_BPSpec2.ps"
retCode = EVLASpectrum(uv, parms["BPCal"], parms["plotTime"], maxgap, plotFile, parms["refAnt"], err, \
Stokes=["RR","LL"], doband=1, \
check=check, debug=debug, logfile=logFile )
if retCode!=0:
raise RuntimeError("Error in Plotting spectrum")
# Amp & phase Recalibrate
if parms["doAmpPhaseCal2"]:
plotFile = fileRoot+"_APCal2.ps"
retCode = KATCalAP (uv, [], parms["ACals"], err, PCals=parms["PCals"], \
doCalib=2, doBand=2, BPVer=1, flagVer=0, \
BChan=parms["ampBChan"], EChan=parms["ampEChan"], \
solInt=parms["solInt"], solSmo=parms["solSmo"], ampScalar=parms["ampScalar"], \
doAmpEdit=True, ampSigma=parms["ampSigma"], \
ampEditFG=parms["ampEditFG"], \
doPlot=parms["doSNPlot"], plotFile=plotFile, refAnt=parms["refAnt"], \
noScrat=noScrat, nThreads=nThreads, logfile=logFile, check=check, debug=debug)
if retCode!=0:
raise RuntimeError("Error calibrating")
# More editing
if parms["doAutoFlag2"]:
mess = "Post recalibration editing:"
printMess(mess, logFile)
retCode = EVLAAutoFlag (uv, [], err, flagVer=0, flagTab=2, \
doCalib=2, gainUse=0, doBand=1, BPVer=1, \
IClip=parms["IClip"], minAmp=parms["minAmp"], timeAvg=parms["timeAvg"], \
doFD=parms["doAFFD"], FDmaxAmp=parms["FDmaxAmp"], FDmaxV=parms["FDmaxV"], \
FDwidMW=parms["FDwidMW"], FDmaxRMS=parms["FDmaxRMS"], \
FDmaxRes=parms["FDmaxRes"], FDmaxResBL= parms["FDmaxResBL"], \
FDbaseSel=parms["FDbaseSel"], \
nThreads=nThreads, logfile=logFile, check=check, debug=debug)
if retCode!=0:
raise RuntimeError("Error in AutoFlag")
# end recal
# Calibrate and average data
if parms["doCalAvg"]:
retCode = KATCalAvg (uv, avgClass, parms["seq"], parms["CalAvgTime"], err, \
flagVer=2, doCalib=2, gainUse=0, doBand=1, BPVer=1, doPol=False, \
avgFreq=parms["avgFreq"], chAvg=parms["chAvg"], \
BChan=1, EChan=parms["selChan"] - 1, doAuto=parms["doAuto"], \
BIF=parms["CABIF"], EIF=parms["CAEIF"], Compress=parms["Compress"], \
nThreads=nThreads, logfile=logFile, check=check, debug=debug)
if retCode!=0:
raise RuntimeError("Error in CalAvg")
# Get calibrated/averaged data
if not check:
uv = UV.newPAUV("AIPS UV DATA", EVLAAIPSName(project), avgClass[0:6], \
disk, parms["seq"], True, err)
if err.isErr:
OErr.printErrMsg(err, "Error creating cal/avg AIPS data")
KATUVFITS(uv, 'preimage.uvfits', 0, err, exclude=["AIPS HI", "AIPS SL", "AIPS PL"], include=["AIPS AN", "AIPS FQ"], compress=parms["Compress"], logfile=logFile)
KATUVFITab(uv, 'preimage.uvtab', 0, err)
# XClip
if parms["XClip"] and parms["XClip"]>0.0:
mess = "Cross Pol clipping:"
printMess(mess, logFile)
retCode = EVLAAutoFlag (uv, [], err, flagVer=-1, flagTab=1, \
doCalib=2, gainUse=0, doBand=-1, maxBad=1.0, \
XClip=parms["XClip"], timeAvg=1./60., \
nThreads=nThreads, logfile=logFile, check=check, debug=debug)
if retCode!=0:
raise RuntimeError("Error in AutoFlag")
# R-L delay calibration cal if needed,
if parms["doRLDelay"] and parms["RLDCal"][0][0]!=None:
if parms["rlrefAnt"]<=0:
parms["rlrefAnt"] = parms["refAnt"]
# parms["rlDoBand"] if before average, BPVer=parms["rlBPVer"],
retCode = EVLARLDelay(uv, err,\
RLDCal=parms["RLDCal"], BChan=parms["rlBChan"], \
EChan=parms["rlEChan"], UVRange=parms["rlUVRange"], \
soucode=parms["rlCalCode"], doCalib=parms["rlDoCal"], gainUse=parms["rlgainUse"], \
timerange=parms["rltimerange"], \
# NOT HERE doBand=parms["rlDoBand"], BPVer=parms["rlBPVer"], \
flagVer=parms["rlflagVer"], \
refAnt=parms["rlrefAnt"], doPol=False, \
nThreads=nThreads, noScrat=noScrat, logfile=logFile, \
check=check, debug=debug)
if retCode!=0:
raise RuntimeError("Error in R-L delay calibration")
# Polarization calibration
if parms["doPolCal"]:
if parms["PCRefAnt"]<=0:
parms["PCRefAnt"] = parms["refAnt"]
retCode = EVLAPolCal(uv, parms["PCInsCals"], err, \
doCalib=2, gainUse=0, doBand=-1, flagVer=0, \
fixPoln=parms["PCFixPoln"], pmodel=parms["PCpmodel"], avgIF=parms["PCAvgIF"], \
solInt=parms["PCSolInt"], refAnt=parms["PCRefAnt"], solType=parms["PCSolType"], \
ChInc=parms["PCChInc"], ChWid=parms["PCChWid"], \
nThreads=nThreads, check=check, debug=debug, noScrat=noScrat, logfile=logFile)
if retCode!=0 and (not check):
raise RuntimeError("Error in polarization calibration: "+str(retCode))
# end poln cal.
# R-L phase calibration cal., creates new BP table
if parms["doRLCal"] and parms["RLDCal"][0][0]!=None:
plotFile = fileRoot+"_RLSpec2.ps"
if parms["rlrefAnt"]<=0:
parms["rlrefAnt"] = parms["refAnt"]
retCode = EVLARLCal(uv, err,\
RLDCal=parms["RLDCal"], BChan=parms["rlBChan"],
EChan=parms["rlEChan"], UVRange=parms["rlUVRange"], \
ChWid2=parms["rlChWid"], solInt1=parms["rlsolint1"], solInt2=parms["rlsolint2"], \
RLPCal=parms["RLPCal"], RLPhase=parms["RLPhase"], \
RM=parms["RLRM"], CleanRad=parms["rlCleanRad"], \
calcode=parms["rlCalCode"], doCalib=parms["rlDoCal"], gainUse=parms["rlgainUse"], \
timerange=parms["rltimerange"], FOV=parms["rlFOV"], \
doBand=-1, BPVer=1, flagVer=parms["rlflagVer"], \
refAnt=parms["rlrefAnt"], doPol=parms["doPol"], PDVer=parms["PDVer"], \
doPlot=parms["doSpecPlot"], plotFile=plotFile, \
nThreads=nThreads, noScrat=noScrat, logfile=logFile, \
check=check, debug=debug)
if retCode!=0:
raise RuntimeError("Error in RL phase spectrum calibration")
# VClip
if parms["VClip"] and parms["VClip"]>0.0:
mess = "VPol clipping:"
printMess(mess, logFile)
retCode = EVLAAutoFlag (uv, [], err, flagVer=-1, flagTab=1, \
doCalib=2, gainUse=0, doBand=-1, \
VClip=parms["VClip"], timeAvg=parms["timeAvg"], \
nThreads=nThreads, logfile=logFile, check=check, debug=debug)
if retCode!=0:
raise RuntimeError("Error in AutoFlag VClip")
# Plot corrected data?
parms["doSpecPlot"]=True
if parms["doSpecPlot"] and parms["plotSource"]:
plotFile = fileRoot+"_Spec.ps"
retCode = EVLASpectrum(uv, parms["BPCal"], parms["plotTime"], maxgap, \
plotFile, parms["refAnt"], err, \
Stokes=["I"], doband=-1, \
check=check, debug=debug, logfile=logFile )
if retCode!=0:
raise RuntimeError("Error in Plotting spectrum")
# Image targets
if parms["doImage"]:
# If targets not specified, image all
if len(parms["targets"])<=0:
slist = EVLAAllSource(uv,err,logfile=logFile,check=check,debug=debug)
else:
slist = targets
slist=targets
KATImageTargets (uv, err, Sources=slist, seq=parms["seq"], sclass=outIClass, OutlierArea=parms["outlierArea"],\
doCalib=-1, doBand=-1, flagVer=-1, doPol=parms["doPol"], PDVer=parms["PDVer"], \
Stokes=parms["Stokes"], FOV=parms["FOV"], Robust=parms["Robust"], Niter=parms["Niter"], \
CleanRad=parms["CleanRad"], minFlux=parms["minFlux"], OutlierSize=parms["OutlierSize"], \
xCells=parms["xCells"], yCells=parms["yCells"], Reuse=parms["Reuse"], minPatch=parms["minPatch"], \
maxPSCLoop=parms["maxPSCLoop"], minFluxPSC=parms["minFluxPSC"], noNeg=parms["noNeg"], \
solPInt=parms["solPInt"], solPMode=parms["solPMode"], solPType=parms["solPType"], \
maxASCLoop=parms["maxASCLoop"], minFluxASC=parms["minFluxASC"], nx=parms["nx"], ny=parms["ny"], \
solAInt=parms["solAInt"], solAMode=parms["solAMode"], solAType=parms["solAType"], \
avgPol=parms["avgPol"], avgIF=parms["avgIF"], minSNR = parms["minSNR"], refAnt=parms["refAnt"], \
do3D=parms["do3D"], BLFact=parms["BLFact"], BLchAvg=parms["BLchAvg"], \
doMB=parms["doMB"], norder=parms["MBnorder"], maxFBW=parms["MBmaxFBW"], \
PBCor=parms["PBCor"],antSize=parms["antSize"], autoCen=parms["autoCen"], \
nTaper=parms["nTaper"], Tapers=parms["Tapers"], sefd=sefd, \
nThreads=nThreads, noScrat=noScrat, logfile=logFile, check=check, debug=False)
# End image
# Get report on sources
if parms["doReport"]:
# If targets not specified, do all
if len(parms["targets"])<=0:
slist = EVLAAllSource(uv,err,logfile=logFile,check=check,debug=debug)
else:
slist = parms["targets"]
Report = EVLAReportTargets(uv, err, Sources=slist, seq=parms["seq"], sclass=outIClass, \
Stokes=parms["Stokes"], logfile=logFile, check=check, debug=debug)
# Save to pickle jar
ReportPicklefile = fileRoot+"_Report.pickle" # Where results saved
SaveObject(Report, ReportPicklefile, True)
# Write results, cleanup
# Save cal/average UV data?
if parms["doSaveUV"] and (not check):
Aname = EVLAAIPSName(project)
cno = AIPSDir.PTestCNO(disk, user, Aname, avgClass[0:6], "UV", parms["seq"], err)
if cno>0:
uvt = UV.newPAUV("AIPS CAL UV DATA", Aname, avgClass, disk, parms["seq"], True, err)
filename = fileRoot+"_Cal.uvtab"
KATUVFITS (uv, filename, 0, err, exclude=["AIPS HI", "AIPS SL", "AIPS PL"], include=["AIPS AN", "AIPS FQ"], compress=parms["Compress"], logfile=logFile)
EVLAAddOutFile(os.path.basename(filename), 'project', "Calibrated Averaged UV data" )
# Save list of output files
EVLASaveOutFiles(manifestfile)
del uvt
# Imaging results
# If targets not specified, save all
if len(parms["targets"])<=0:
slist = EVLAAllSource(uv,err,logfile=logFile,check=check,debug=debug)
else:
slist = parms["targets"]
for target in slist:
if parms["doSaveImg"] and (not check):
for s in parms["Stokes"]:
oclass = s+outIClass[1:]
outname = target
# Test if image exists
cno = AIPSDir.PTestCNO(disk, user, outname, oclass, "MA", parms["seq"], err)
#print cno
if cno <= 0 :
continue
x = Image.newPAImage("out", outname, oclass, disk, parms["seq"], True, err)
outfilefits = fileRoot+'_'+target+"."+oclass+".fits"
xf = KATImFITS(x, outfilefits, 0, err, logfile=logFile)
x = Image.newPAImage("out", outname, oclass, disk, parms["seq"], True, err)
outfile = fileRoot+'_'+target+"."+oclass+".fittab.fits"
xf = EVLAImFITS (x, outfile, 0, err, logfile=logFile)
EVLAAddOutFile(outfile, target, 'Image of '+ target)
# Statistics
zz=imstat(x, err, logfile=logFile)
# Make a Jpeg image
FITS2jpeg.fits2jpeg(outfilefits,chans=1,contrast=0.05,cmap='jet',area=0.7)
EVLAAddOutFile(outfile.replace('.fits','.jpeg'), target, 'Jpeg image of '+ target)
# end writing loop
# Save list of output files
EVLASaveOutFiles(manifestfile)
OErr.printErrMsg(err, "Writing output")
# Contour plots
if parms["doKntrPlots"]:
mess = "INFO --> Contour plots (doKntrPlots)"
printMess(mess, logFile)
EVLAKntrPlots( err, imName=parms["targets"], project=fileRoot,
disk=disk, debug=debug )
# Save list of output files
EVLASaveOutFiles(manifestfile)
elif debug:
mess = "Not creating contour plots ( doKntrPlots = "+str(parms["doKntrPlots"])+ " )"
printMess(mess, logFile)
# Source uv plane diagnostic plots
if parms["doDiagPlots"]:
mess = "INFO --> Diagnostic plots (doDiagPlots)"
printMess(mess, logFile)
# Get the highest number avgClass catalog file
Aname = EVLAAIPSName( project )
uvc = None
if not check:
uvname = project+"_Cal"
uvc = UV.newPAUV(uvname, Aname, avgClass, disk, parms["seq"], True, err)
EVLADiagPlots( uvc, err, cleanUp=parms["doCleanup"], \
project=fileRoot, \
logfile=logFile, check=check, debug=debug )
# Save list of output files
EVLASaveOutFiles(manifestfile)
elif debug:
mess = "Not creating diagnostic plots ( doDiagPlots = "+str(parms["doDiagPlots"])+ " )"
printMess(mess, logFile)
# Save metadata
srcMetadata = None
projMetadata = None
if parms["doMetadata"]:
mess = "INFO --> Save metadata (doMetadata)"
printMess(mess, logFile)
uvc = None
if not uvc:
# Get calibrated/averaged data
Aname = EVLAAIPSName(project)
uvname = project+"_Cal"
uvc = UV.newPAUV(uvname, Aname, avgClass, disk, parms["seq"], True, err)
if err.isErr:
OErr.printErrMsg(err, "Error creating cal/avg AIPS data")
# Get source metadata; save to pickle file
srcMetadata = EVLASrcMetadata( uvc, err, Sources=parms["targets"], seq=parms["seq"], \
sclass=outIClass, Stokes=parms["Stokes"],\
logfile=logFile, check=check, debug=debug )
picklefile = fileRoot+".SrcReport.pickle"
SaveObject( srcMetadata, picklefile, True )
EVLAAddOutFile(os.path.basename(picklefile), 'project', 'All source metadata' )
# Get project metadata; save to pickle file
projMetadata = KATProjMetadata( uvc, AIPS_VERSION, err, \
PCals=parms["PCals"], ACals=parms["ACals"], \
BPCals=parms["BPCals"], DCals=parms["DCals"], \
project = project, band = band, \
dataInUVF = parms["archRoot"], archFileID = fileRoot )
picklefile = fileRoot+".ProjReport.pickle"
SaveObject(projMetadata, picklefile, True)
EVLAAddOutFile(os.path.basename(picklefile), 'project', 'Project metadata' )
else:
# Fetch from pickle jar
picklefile = fileRoot+".SrcReport.pickle"
srcMetadata = FetchObject(picklefile)
picklefile = fileRoot+".ProjReport.pickle"
projMetadata = FetchObject(picklefile)
# Write report
if parms["doHTML"]:
mess = "INFO --> Write HTML report (doHTML)"
printMess(mess, logFile)
KATHTMLReport( projMetadata, srcMetadata, \
outfile=fileRoot+"_report.html", \
logFile=logFile )
# Write VOTable
if parms["doVOTable"]:
mess = "INFO --> Write VOTable (doVOTable)"
printMess(mess, logFile)
EVLAAddOutFile( 'VOTable.xml', 'project', 'VOTable report' )
EVLAWriteVOTable( projMetadata, srcMetadata, filename=fileRoot+'_VOTable.xml' )
# Save list of output files
EVLASaveOutFiles(manifestfile)
# Cleanup - delete AIPS files
if parms["doCleanup"] and (not check):
mess = "INFO --> Clean up (doCleanup)"
printMess(mess, logFile)
# Delete target images
# How many Stokes images
nstok = len(parms["Stokes"])
for istok in range(0,nstok):
oclass = parms["Stokes"][istok:istok+1]+outIClass[1:]
AllDest(err, disk=disk,Aseq=parms["seq"],Aclass=oclass)
# Delete initial UV data
Aname = EVLAAIPSName(project)
# Test if data exists
cno = AIPSDir.PTestCNO(disk, user, Aname, dataClass[0:6], "UV", parms["seq"], err)
if cno>0:
uvt = UV.newPAUV("AIPS RAW UV DATA", Aname, dataClass[0:6], disk, parms["seq"], True, err)
uvt.Zap(err)
del uvt
if err.isErr:
OErr.printErrMsg(err, "Error deleting raw AIPS data")
# Zap calibrated/averaged data
# Test if data exists
cno = AIPSDir.PTestCNO(disk, user, Aname, avgClass[0:6], "UV", parms["seq"], err)
if cno>0:
uvt = UV.newPAUV("AIPS CAL UV DATA", Aname, avgClass[0:6], disk, parms["seq"], True, err)
uvt.Zap(err)
del uvt
if err.isErr:
OErr.printErrMsg(err, "Error deleting cal/avg AIPS data")
# Zap UnHanned data if present
loadClass = "Raw"
# Test if image exists
cno = AIPSDir.PTestCNO(disk, user, Aname, loadClass[0:6], "UV", parms["seq"], err)
if cno>0:
uvt = UV.newPAUV("AIPS CAL UV DATA", Aname, loadClass[0:6], disk, parms["seq"], True, err)
uvt.Zap(err)
del uvt
if err.isErr:
OErr.printErrMsg(err, "Error deleting cal/avg AIPS data")
OErr.printErrMsg(err, "Writing output/cleanup")
# Delete AIPS scratch DA00 and disk
if os.path.exists(os.environ['DA00']): shutil.rmtree(os.environ['DA00'])
for disk in ObitTalkUtil.AIPSDir.AIPSdisks:
if os.path.exists(disk): shutil.rmtree(disk)
# Shutdown
mess = "Finished project "+parms["project"]+ \
" AIPS user no. "+str(AIPS.userno)
printMess(mess, logFile)
OErr.printErr(err)
OSystem.Shutdown(ObitSys)
def MKContPipeline(files, outputdir, **kwargs):
"""MeerKAT Continuum pipeline.
Parameters
----------
files : list
h5 filenames (note: support for multiple h5 files
i.e. ConcatenatedDataSet is not currently supported)
outputdir : string
Directory location to write output data,
scratchdir : string, optional
The directory location of the aips disk
parmFile : string, optional
Overwrite the default imaging parameters using this parameter file.
"""
#if len(files) > 1:
# raise TooManyKatfilesException('Processing multiple katfiles are not currently supported')
# Onle be concatenated if we have to be
if len(files) == 1:
h5file = files[0]
else:
h5file = files
# Die gracefully if we cannot write to the output area...
if not os.path.exists(outputdir):
print('Specified output directory: ' + outputdir + 'does not exist.')
exit(-1)
# Obit error logging
err = OErr.OErr()
#################### Initialize filenames #######################################################
fileRoot = os.path.join(outputdir,
os.path.basename(os.path.splitext(
files[0])[0])) # root of file name
logFile = fileRoot + ".log" # Processing log file
avgClass = ("UVAv")[0:6] # Averaged data AIPS class
manifestfile = outputdir + '/manifest.pickle'
############################# Initialize OBIT and AIPS ##########################################
noScrat = []
# Logging directly to logFile
OErr.PInit(err, 2, logFile)
EVLAAddOutFile(os.path.basename(logFile), 'project', 'Pipeline log file')
if kwargs.get('reuse'):
ObitSys = AIPSSetup.AIPSSetup(err,
configfile=kwargs.get('configFile'),
scratchdir=kwargs.get('scratchdir'),
aipsdisk=kwargs.get('aipsdisk'),
overwrite=False)
else:
ObitSys = AIPSSetup.AIPSSetup(err,
configfile=kwargs.get('configFile'),
scratchdir=kwargs.get('scratchdir'),
aipsdisk=kwargs.get('aipsdisk'))
# Get the set up AIPS environment.
AIPS_ROOT = os.environ['AIPS_ROOT']
AIPS_VERSION = os.environ['AIPS_VERSION']
nThreads = 72
user = OSystem.PGetAIPSuser()
AIPS.userno = user
disk = 1
fitsdisk = 0
nam = os.path.basename(os.path.splitext(files[0])[0])[0:10]
cls = "Raw"
seq = 1
############################# Initialise Parameters ##########################################
####### Initialize parameters dictionary #####
parms = KATInitContParms()
####### User defined parameters ######
if kwargs.get('parmFile'):
print("parmFile", kwargs.get('parmFile'))
exec(open(kwargs.get('parmFile')).read())
EVLAAddOutFile(os.path.basename(kwargs.get('parmFile')), 'project',
'Pipeline input parameters')
############### Initialize katfile object, uvfits object and condition data #########################
OK = False
# Open the h5 file as a katfile object
try:
#open katfile and perform selection according to kwargs
katdata = katfile.open(h5file)
OK = True
except Exception as exception:
print(exception)
if not OK:
OErr.PSet(err)
OErr.PLog(err, OErr.Fatal,
"Unable to read KAT HDF5 data in " + str(h5file))
raise KATUnimageableError("Unable to read KAT HDF5 data in " +
str(h5file))
#Are we MeerKAT or KAT-7
telescope = katdata.ants[0].name[0]
if telescope == 'm':
sefd = 500.
else:
sefd = 1200.
#Get calibrator models
fluxcals = katpoint.Catalogue(
open(FITSDir.FITSdisks[0] + "/" + parms["fluxModel"]))
#Condition data (get bpcals, update names for aips conventions etc)
KATh5Condition(katdata, fluxcals, err)
###################### Data selection and static edits ############################################
# Select data based on static imageable parameters
MKATh5Select(katdata, parms, err, **kwargs)
# General AIPS data parameters at script level
dataClass = ("UVDa")[0:6] # AIPS class of raw uv data
band = katdata.spectral_windows[0].product #Correlator product
project = os.path.basename(os.path.splitext(files[0])[0])[
0:10] # Project name (12 char or less, used as AIPS Name)
outIClass = parms["outIClass"] # image AIPS class
debug = parms["debug"]
check = parms["check"]
####################### Import data into AIPS #####################################################
# Reuse or nay?
if kwargs.get('reuse'):
uv = UV.newPAUV("AIPS UV DATA", EVLAAIPSName(project), dataClass, disk,
seq, True, err)
obsdata = KATH5toAIPS.GetKATMeta(katdata, err)
# Extract AIPS parameters of the uv data to the metadata
obsdata["Aproject"] = uv.Aname
obsdata["Aclass"] = uv.Aclass
obsdata["Aseq"] = uv.Aseq
obsdata["Adisk"] = disk
obsdata["calInt"] = katdata.dump_period
obsdata["fitsdisk"] = fitsdisk
# TODO: Check if the input data has been Hanned.
doneHann = True
else:
# Number of baselines gives batch size
nbl = len(
np.unique([(cp[0][:-1] + cp[1][:-1]).upper()
for cp in katdata.corr_products]))
# Construct a template uvfits file from master template
mastertemplate = ObitTalkUtil.FITSDir.FITSdisks[
fitsdisk] + 'MKATTemplate.uvtab.gz'
outtemplate = nam + '.uvtemp'
KATH5toAIPS.MakeTemplate(mastertemplate,
outtemplate,
len(katdata.channel_freqs),
nvispio=nbl)
uv = OTObit.uvlod(outtemplate, 0, EVLAAIPSName(project), cls, disk,
seq, err)
obsdata = KATH5toAIPS.KAT2AIPS(katdata,
uv,
disk,
fitsdisk,
err,
calInt=katdata.dump_period,
**kwargs)
MakeIFs.UVMakeIF(uv, 8, err)
os.remove(outtemplate)
# Print the uv data header to screen.
uv.Header(err)
############################# Set Project Processing parameters ###################################
# Parameters derived from obsdata and katdata
MKATGetObsParms(obsdata, katdata, parms, logFile)
###### Initialise target parameters #####
KATInitTargParms(katdata, parms, err)
# Load the outputs pickle jar
EVLAFetchOutFiles()
OSystem.PAllowThreads(nThreads) # Allow threads in Obit/oython
retCode = 0
maxgap = max(parms["CalAvgTime"], 20 * katdata.dump_period) / 60.
################### Start processing ###############################################################
mess = "Start project "+parms["project"]+" AIPS user no. "+str(AIPS.userno)+\
", KAT7 configuration "+parms["KAT7Cfg"]
printMess(mess, logFile)
if debug:
pydoc.ttypager = pydoc.plainpager # don't page task input displays
mess = "Using Debug mode "
printMess(mess, logFile)
if check:
mess = "Only checking script"
printMess(mess, logFile)
# Log parameters
printMess("Parameter settings", logFile)
for p in parms:
mess = " " + p + ": " + str(parms[p])
printMess(mess, logFile)
clist = []
for DCal in parms["DCals"]:
if DCal["Source"] not in clist:
clist.append(DCal["Source"])
for PCal in parms["PCals"]:
if PCal["Source"] not in clist:
clist.append(PCal["Source"])
for ACal in parms["ACals"]:
if ACal["Source"] not in clist:
clist.append(ACal["Source"])
if kwargs.get('targets') is not None:
targets = [
targ.name for targ in katdata.catalogue
if (targ.name not in clist) and (
targ.name in kwargs.get('targets').split(','))
]
else:
targets = [
targ.name for targ in katdata.catalogue if (targ.name not in clist)
]
refAnt = FetchObject(fileRoot + ".refAnt.pickle")
# Save parameters to pickle jar, manifest
ParmsPicklefile = fileRoot + ".Parms.pickle" # Where results saved
SaveObject(parms, ParmsPicklefile, True)
EVLAAddOutFile(os.path.basename(ParmsPicklefile), 'project',
'Processing parameters used')
loadClass = dataClass
# Hanning - No Hanning
parms["doHann"] = False
doneHann = False
if doneHann:
# Halve channels after hanning.
parms["selChan"] = int(parms["selChan"] / 2)
parms["BChDrop"] = int(parms["BChDrop"] / 2)
parms["EChDrop"] = int(parms["EChDrop"] / 2)
if uv == None and not check:
raise RuntimeError("Cannot Hann data ")
# Clear any old calibration/editing
if parms["doClearTab"] or kwargs.get('reuse'):
mess = "Clear previous calibration"
printMess(mess, logFile)
EVLAClearCal(uv,
err,
doGain=parms["doClearGain"],
doFlag=parms["doClearFlag"],
doBP=parms["doClearBP"],
check=check)
OErr.printErrMsg(err, "Error resetting calibration")
# Quack to remove data from start and end of each scan
if parms["doQuack"]:
retCode = EVLAQuack (uv, err, begDrop=parms["quackBegDrop"], endDrop=parms["quackEndDrop"], \
Reason=parms["quackReason"], \
logfile=logFile, check=check, debug=debug)
if retCode != 0:
raise RuntimeError("Error Quacking data")
# Flag antennas shadowed by others?
if parms["doShad"]:
retCode = EVLAShadow (uv, err, shadBl=parms["shadBl"], \
logfile=logFile, check=check, debug=debug)
if retCode != 0:
raise RuntimeError("Error Shadow flagging data")
# Median window time editing, for RFI impulsive in time
if parms["doMednTD1"]:
mess = "Median window time editing, for RFI impulsive in time:"
printMess(mess, logFile)
retCode = EVLAMedianFlag (uv, clist, err, noScrat=noScrat, nThreads=nThreads, \
avgTime=parms["mednAvgTime"], avgFreq=parms["mednAvgFreq"], chAvg= parms["mednChAvg"], \
timeWind=parms["mednTimeWind"],flagVer=2, flagTab=2,flagSig=parms["mednSigma"], \
logfile=logFile, check=check, debug=False)
if retCode != 0:
raise RuntimeError("Error in MednFlag")
# Median window frequency editing, for RFI impulsive in frequency
if parms["doFD1"]:
mess = "Median window frequency editing, for RFI impulsive in frequency:"
printMess(mess, logFile)
retCode = EVLAAutoFlag (uv, clist, err, flagVer=2, flagTab=2, doCalib=-1, doBand=-1, \
timeAvg=parms["FD1TimeAvg"], \
doFD=True, FDmaxAmp=1.0e20, FDmaxV=1.0e20, FDwidMW=parms["FD1widMW"], \
FDmaxRMS=[1.0e20,0.1], FDmaxRes=parms["FD1maxRes"], \
FDmaxResBL= parms["FD1maxRes"], FDbaseSel=parms["FD1baseSel"],\
nThreads=nThreads, logfile=logFile, check=check, debug=debug)
if retCode != 0:
raise RuntimeError("Error in AutoFlag")
# Parallactic angle correction?
if parms["doPACor"]:
retCode = EVLAPACor(uv, err, \
logfile=logFile, check=check, debug=debug)
if retCode != 0:
raise RuntimeError("Error in Parallactic angle correction")
# Need to find a reference antenna? See if we have saved it?
if (parms["refAnt"] <= 0):
refAnt = FetchObject(fileRoot + ".refAnt.pickle")
if refAnt:
parms["refAnt"] = refAnt
# Use bandpass calibrator and center half of each spectrum
if parms["refAnt"] <= 0:
mess = "Find best reference antenna: run Calib on BP Cal(s) "
printMess(mess, logFile)
parms["refAnt"] = EVLAGetRefAnt(uv, parms["BPCals"], err, flagVer=0, \
solInt=parms["bpsolint1"], nThreads=nThreads, \
logfile=logFile, check=check, debug=debug)
if err.isErr:
raise RuntimeError("Error finding reference antenna")
if parms["refAnts"][0] <= 0:
parms["refAnts"][0] = parms["refAnt"]
mess = "Picked reference antenna " + str(parms["refAnt"])
printMess(mess, logFile)
# Save it
ParmsPicklefile = fileRoot + ".Parms.pickle" # Where results saved
SaveObject(parms, ParmsPicklefile, True)
refAntPicklefile = fileRoot + ".refAnt.pickle" # Where results saved
SaveObject(parms["refAnt"], refAntPicklefile, True)
# Plot Raw, edited data?
parms["doRawSpecPlot"] = False
parms["doSpecPlot"] = False
if parms["doRawSpecPlot"] and parms["plotSource"]:
mess = "Raw Spectral plot for: " + ' '.join(parms["BPCal"])
printMess(mess, logFile)
plotFile = fileRoot + "_RawSpec.ps"
retCode = EVLASpectrum(uv, parms["BPCal"], parms["plotTime"], maxgap, plotFile, parms["refAnt"], err, \
Stokes=["RR","LL"], doband=-1, \
check=check, debug=debug, logfile=logFile )
if retCode != 0:
raise RuntimeError("Error in Plotting spectrum")
EVLAAddOutFile(plotFile, 'project', 'Pipeline log file')
# delay calibration
if parms["doDelayCal"] and parms["DCals"] and not check:
plotFile = fileRoot + "_DelayCal.ps"
retCode = EVLADelayCal(uv, parms["DCals"], err, \
BChan=parms["delayBChan"], EChan=parms["delayEChan"], \
doCalib=2, flagVer=0, doBand=-1, \
solInt=parms["delaySolInt"], smoTime=parms["delaySmoo"], \
refAnts=[parms["refAnt"]], doTwo=parms["doTwo"],
doZeroPhs=parms["delayZeroPhs"], \
doPlot=parms["doSNPlot"], plotFile=plotFile, \
nThreads=nThreads, noScrat=noScrat, \
logfile=logFile, check=check, debug=debug)
if retCode != 0:
raise RuntimeError("Error in delay calibration")
# Plot corrected data?
if parms["doSpecPlot"] and parms["plotSource"]:
plotFile = fileRoot + "_DelaySpec.ps"
retCode = EVLASpectrum(uv, parms["BPCal"], parms["plotTime"], maxgap, \
plotFile, parms["refAnt"], err, \
Stokes=["RR","LL"], doband=-1, \
check=check, debug=debug, logfile=logFile )
if retCode != 0:
raise RuntimeError("Error in Plotting spectrum")
print(parms["bpBChan1"], parms["bpEChan1"], parms["bpBChan2"],
parms["bpEChan2"], parms["bpChWid2"])
# Bandpass calibration
if parms["doBPCal"] and parms["BPCals"]:
retCode = KATBPCal(uv, parms["BPCals"], err, noScrat=noScrat, solInt1=parms["bpsolint1"], \
solInt2=parms["bpsolint2"], solMode=parms["bpsolMode"], \
BChan1=parms["bpBChan1"], EChan1=parms["bpEChan1"], \
BChan2=parms["bpBChan2"], EChan2=parms["bpEChan2"], ChWid2=parms["bpChWid2"], \
doCenter1=parms["bpDoCenter1"], refAnt=parms["refAnt"], \
UVRange=parms["bpUVRange"], doCalib=2, gainUse=0, flagVer=0, doPlot=False, \
nThreads=nThreads, logfile=logFile, check=check, debug=debug)
if retCode != 0:
raise RuntimeError("Error in Bandpass calibration")
# Plot corrected data?
if parms["doSpecPlot"] and parms["plotSource"]:
plotFile = fileRoot + "_BPSpec.ps"
retCode = EVLASpectrum(uv, parms["BPCal"], parms["plotTime"], maxgap, plotFile, \
parms["refAnt"], err, Stokes=["RR","LL"], doband=2, \
check=check, debug=debug, logfile=logFile )
if retCode != 0:
raise RuntimeError("Error in Plotting spectrum")
# Amp & phase Calibrate
if parms["doAmpPhaseCal"]:
plotFile = fileRoot + "_APCal.ps"
retCode = KATCalAP (uv, [], parms["ACals"], err, PCals=parms["PCals"],
doCalib=2, doBand=2, BPVer=1, flagVer=0, \
BChan=parms["ampBChan"], EChan=parms["ampEChan"], \
solInt=parms["solInt"], solSmo=parms["solSmo"], ampScalar=parms["ampScalar"], \
doAmpEdit=parms["doAmpEdit"], ampSigma=parms["ampSigma"], \
ampEditFG=parms["ampEditFG"], \
doPlot=parms["doSNPlot"], plotFile=plotFile, refAnt=parms["refAnt"], \
nThreads=nThreads, noScrat=noScrat, logfile=logFile, check=check, debug=debug)
#print parms["ACals"],parms["PCals"]
if retCode != 0:
raise RuntimeError("Error calibrating")
# More editing
if parms["doAutoFlag"]:
mess = "Post calibration editing:"
printMess(mess, logFile)
# if going to redo then only calibrators
if parms["doRecal"]:
# Only calibrators
clist = []
for DCal in parms["DCals"]:
if DCal["Source"] not in clist:
clist.append(DCal["Source"])
for PCal in parms["PCals"]:
if PCal["Source"] not in clist:
clist.append(PCal["Source"])
for ACal in parms["ACals"]:
if ACal["Source"] not in clist:
clist.append(ACal["Source"])
else:
clist = []
retCode = EVLAAutoFlag (uv, clist, err, flagVer=0, flagTab =2, \
doCalib=2, gainUse=0, doBand=2, BPVer=1, \
IClip=parms["IClip"], minAmp=parms["minAmp"], timeAvg=parms["timeAvg"], \
doFD=parms["doFirstAFFD"], FDmaxAmp=parms["FDmaxAmp"], FDmaxV=parms["FDmaxV"], \
FDwidMW=parms["FDwidMW"], FDmaxRMS=parms["FDmaxRMS"], \
FDmaxRes=parms["FDmaxRes"], FDmaxResBL=parms["FDmaxResBL"], \
FDbaseSel=parms["FDbaseSel"], \
nThreads=nThreads, logfile=logFile, check=check, debug=debug)
if retCode != 0:
raise RuntimeError("Error in AutoFlag")
# Redo the calibration using new flagging?
if parms["doBPCal2"] == None:
parms["doBPCal2"] = parms["doBPCal"]
if parms["doDelayCal2"] == None:
parms["doDelayCal2"] = parms["doDelayCal2"]
if parms["doAmpPhaseCal2"] == None:
parms["doAmpPhaseCal2"] = parms["doAmpPhaseCal"]
if parms["doAutoFlag2"] == None:
parms["doAutoFlagCal2"] = parms["doAutoFlag"]
if parms["doRecal"]:
mess = "Redo calibration:"
printMess(mess, logFile)
EVLAClearCal(uv,
err,
doGain=True,
doFlag=False,
doBP=True,
check=check,
logfile=logFile)
OErr.printErrMsg(err, "Error resetting calibration")
# Parallactic angle correction?
if parms["doPACor"]:
retCode = EVLAPACor(uv, err, \
logfile=logFile, check=check, debug=debug)
if retCode != 0:
raise RuntimeError("Error in Parallactic angle correction")
# Delay recalibration
if parms["doDelayCal2"] and parms["DCals"] and not check:
plotFile = fileRoot + "_DelayCal2.ps"
retCode = EVLADelayCal(uv, parms["DCals"], err, \
BChan=parms["delayBChan"], EChan=parms["delayEChan"], \
doCalib=2, flagVer=0, doBand=-1, \
solInt=parms["delaySolInt"], smoTime=parms["delaySmoo"], \
refAnts=[parms["refAnt"]], doTwo=parms["doTwo"], \
doZeroPhs=parms["delayZeroPhs"], \
doPlot=parms["doSNPlot"], plotFile=plotFile, \
nThreads=nThreads, noScrat=noScrat, \
logfile=logFile, check=check, debug=debug)
if retCode != 0:
raise RuntimeError("Error in delay calibration")
# Plot corrected data?
if parms["doSpecPlot"] and parms["plotSource"]:
plotFile = fileRoot + "_DelaySpec2.ps"
retCode = EVLASpectrum(uv, parms["BPCal"], parms["plotTime"], maxgap, plotFile, parms["refAnt"], err, \
Stokes=["RR","LL"], doband=-1, \
check=check, debug=debug, logfile=logFile )
if retCode != 0:
raise RuntimeError("Error in Plotting spectrum")
# Bandpass calibration
if parms["doBPCal2"] and parms["BPCals"]:
retCode = KATBPCal(uv, parms["BPCals"], err, noScrat=noScrat, solInt1=parms["bpsolint1"], \
solInt2=parms["bpsolint2"], solMode=parms["bpsolMode"], \
BChan1=parms["bpBChan1"], EChan1=parms["bpEChan1"], \
BChan2=parms["bpBChan2"], EChan2=parms["bpEChan2"], ChWid2=parms["bpChWid2"], \
doCenter1=parms["bpDoCenter1"], refAnt=parms["refAnt"], \
UVRange=parms["bpUVRange"], doCalib=2, gainUse=0, flagVer=0, doPlot=False, \
nThreads=nThreads, logfile=logFile, check=check, debug=debug)
if retCode != 0:
raise RuntimeError("Error in Bandpass calibration")
# Plot corrected data?
if parms["doSpecPlot"] and parms["plotSource"]:
plotFile = fileRoot + "_BPSpec2.ps"
retCode = EVLASpectrum(uv, parms["BPCal"], parms["plotTime"], maxgap, plotFile, parms["refAnt"], err, \
Stokes=["RR","LL"], doband=2, \
check=check, debug=debug, logfile=logFile )
if retCode != 0:
raise RuntimeError("Error in Plotting spectrum")
# Amp & phase Recalibrate
if parms["doAmpPhaseCal2"]:
plotFile = fileRoot + "_APCal2.ps"
retCode = KATCalAP (uv, [], parms["ACals"], err, PCals=parms["PCals"], \
doCalib=2, doBand=2, BPVer=1, flagVer=0, \
BChan=parms["ampBChan"], EChan=parms["ampEChan"], \
solInt=parms["solInt"], solSmo=parms["solSmo"], ampScalar=parms["ampScalar"], \
doAmpEdit=True, ampSigma=parms["ampSigma"], \
ampEditFG=parms["ampEditFG"], \
doPlot=parms["doSNPlot"], plotFile=plotFile, refAnt=parms["refAnt"], \
noScrat=noScrat, nThreads=nThreads, logfile=logFile, check=check, debug=debug)
if retCode != 0:
raise RuntimeError("Error calibrating")
# More editing
parms["doAutoFlag2"] = False
if parms["doAutoFlag2"]:
mess = "Post recalibration editing:"
printMess(mess, logFile)
retCode = EVLAAutoFlag (uv, [], err, flagVer=0, flagTab=2, \
doCalib=2, gainUse=0, doBand=2, BPVer=1, \
IClip=parms["IClip"], minAmp=parms["minAmp"], timeAvg=parms["timeAvg"], \
doFD=parms["doSecAFFD"], FDmaxAmp=parms["FDmaxAmp"], FDmaxV=parms["FDmaxV"], \
FDwidMW=parms["FDwidMW"], FDmaxRMS=parms["FDmaxRMS"], \
FDmaxRes=parms["FDmaxRes"], FDmaxResBL= parms["FDmaxResBL"], \
FDbaseSel=parms["FDbaseSel"], \
nThreads=nThreads, logfile=logFile, check=check, debug=debug)
if retCode != 0:
raise RuntimeError("Error in AutoFlag")
# end recal
# Calibrate and average data
# Overwrite avgStokes from command line
parms["avgFreq"] = 0
parms["chAvg"] = 1
parms["doCalAvg"] = 'Splat'
if kwargs.get('halfstokes'):
parms["avgStokes"] = 'HALF'
if parms["doCalAvg"] == 'Splat':
retCode = KATCalAvg (uv, avgClass, parms["seq"], parms["CalAvgTime"], err, \
flagVer=2, doCalib=2, gainUse=0, doBand=2, BPVer=1, doPol=False, \
avgFreq=parms["avgFreq"], chAvg=parms["chAvg"], Stokes=parms["avgStokes"], \
BChan=1, EChan=parms["selChan"] - 1, doAuto=parms["doAuto"], \
BIF=parms["CABIF"], EIF=parms["CAEIF"], Compress=parms["Compress"], \
nThreads=nThreads, logfile=logFile, check=check, debug=debug)
if retCode != 0:
raise RuntimeError("Error in CalAvg")
elif parms["doCalAvg"] == 'BL':
retCode = KATBLCalAvg (uv, avgClass, parms["seq"], err, \
flagVer=2, doCalib=2, gainUse=0, doBand=2, BPVer=1, doPol=False, \
avgFreq=parms["avgFreq"], chAvg=parms["chAvg"], FOV=parms['FOV'], \
maxInt=min(parms["solPInt"],parms["solAInt"]), Stokes=parms["avgStokes"], \
BChan=1, EChan=parms["selChan"] - 1, timeAvg=parms["CalAvgTime"], \
BIF=parms["CABIF"], EIF=parms["CAEIF"], Compress=parms["Compress"], \
logfile=logFile, check=check, debug=debug)
if retCode != 0:
raise RuntimeError("Error in BLCalAvg")
if parms["doSaveTab"]:
filename = project + ".CalTab.uvtab"
_ = EVLAUVFITSTab(uv, filename, 0, err, logfile=logFile)
#Zap unaveraged data if requested
if kwargs.get('zapraw'):
uv.Zap(err)
# Get calibrated/averaged data
if not check:
uv = UV.newPAUV("AIPS UV DATA", EVLAAIPSName(project), avgClass[0:6], \
disk, parms["seq"], True, err)
if err.isErr:
OErr.printErrMsg(err, "Error creating cal/avg AIPS data")
plotFile = fileRoot + "_Spec.ps"
retCode = EVLASpectrum(uv, parms["BPCal"], parms["plotTime"], maxgap, \
plotFile, parms["refAnt"], err, \
Stokes=["I"], doband=-1, docalib=-1, \
check=check, debug=debug, logfile=logFile )
if retCode != 0:
raise RuntimeError("Error in Plotting spectrum")
# KATUVFITS(uv, 'preimage.uvfits', 0, err, exclude=["AIPS HI", "AIPS SL", "AIPS PL"],
# include=["AIPS AN", "AIPS FQ"], compress=parms["Compress"], logfile=logFile)
KATUVFITab(uv, project + '.uvtab', 0, err)
#Gzip the data?
if kwargs.get('gzip'):
os.system('pigz -p %d %s' % (nThreads, project + '.uvtab'))
os.system('rm -f %s' % (project + '.uvtab'))
def VLAUVLoad(filename, inDisk, Aname, Aclass, Adisk, Aseq, err, logfile=''):
""" Read FITS uvtab file into AIPS
Read a UVTAB FITS UV data file and write an AIPS data set
filename = name of FITS file
inDisk = FITS directory number
Aname = AIPS name of file
Aclass = AIPS class of file
Aseq = AIPS sequence number of file, 0=> create new
Adisk = FITS directory number
err = Python Obit Error/message stack
logfile = logfile for messages
returns AIPS UV data object
"""
################################################################
#
# Checks
if not OErr.OErrIsA(err):
raise TypeError, "err MUST be an OErr"
#
# Get input
inUV = UV.newPFUV("FITS UV DATA", filename, inDisk, True, err)
if err.isErr:
OErr.printErrMsg(err, "Error with FITS data")
# Get output, create new if seq=0
if Aseq < 1:
OErr.printErr(err) # Print any outstanding messages
user = OSystem.PGetAIPSuser()
Aseq = AIPSDir.PHiSeq(Adisk, user, Aname, Aclass, "MA", err)
# If it already exists, increment seq
if AIPSDir.PTestCNO(Adisk, user, Aname, Aclass, "MA", Aseq, err) > 0:
Aseq = Aseq + 1
OErr.PClear(err) # Clear any message/error
mess = "Creating AIPS UV file " + Aname + "." + Aclass + "." + str(
Aseq) + " on disk " + str(Adisk)
printMess(mess, logfile)
outUV = UV.newPAUV("AIPS UV DATA", Aname, Aclass, Adisk, Aseq, False, err)
if err.isErr:
OErr.printErrMsg(err, "Error creating AIPS data")
# Copy
UV.PCopy(inUV, outUV, err)
if err.isErr:
OErr.printErrMsg(err, "Error copying UV data to AIPS")
# Copy History
inHistory = History.History("inhistory", inUV.List, err)
outHistory = History.History("outhistory", outUV.List, err)
History.PCopyHeader(inHistory, outHistory, err)
# Add history
outHistory.Open(History.READWRITE, err)
outHistory.TimeStamp(" Start Obit uvlod", err)
outHistory.WriteRec(
-1, "uvlod / FITS file " + filename + " disk " + str(inDisk), err)
outHistory.Close(err)
#
# Copy Tables
exclude = [
"AIPS HI", "AIPS AN", "AIPS FQ", "AIPS SL", "AIPS PL", "History"
]
include = []
UV.PCopyTables(inUV, outUV, exclude, include, err)
return outUV # return new object
def MKContPipeline(files, outputdir, **kwargs):
"""MeerKAT Continuum pipeline.
Parameters
----------
files : list
h5 filenames (note: support for multiple h5 files
i.e. ConcatenatedDataSet is not currently supported)
outputdir : string
Directory location to write output data,
scratchdir : string, optional
The directory location of the aips disk
parmFile : string, optional
Overwrite the default imaging parameters using this parameter file.
"""
if len(files) == 1:
h5file = files[0]
else:
h5file = files
############### Initialize katfile object #########################
OK = False
# Open the h5 file as a katfile object
try:
#open katfile and perform selection according to kwargs
katdal_ref_ant = kwargs.get('katdal_refant', '')
katdal_retries = kwargs.get('katdal_retries', 2)
katdal_timeout = kwargs.get('katdal_timeout', 300)
katdata = katfile.open(h5file,
ref_ant=katdal_ref_ant,
timeout=katdal_timeout,
retries=katdal_retries)
OK = True
except Exception as exception:
print(exception)
if not OK:
raise KATUnimageableError("Unable to read MVF data in " + str(h5file))
# If we are doing polcal- search for the most recent delaycal observation
if kwargs.get('polcal'):
if kwargs.get('delaycal_mvf') is None:
# Automatically determine delay_cal CBID
delay_katdata = KATGetDelayCal(h5file,
katdata,
timeout=katdal_timeout,
retries=katdal_retries)
else:
# Use the user supplied one
delay_katdata = KATGetDelayCal(kwargs.get('delaycal_mvf'))
kwargs["delay_katdata"] = delay_katdata
# Die gracefully if we cannot write to the output area...
if not os.path.exists(outputdir):
print('Specified output directory: ' + outputdir + 'does not exist.')
exit(-1)
# Obit error logging
err = OErr.OErr()
#################### Initialize filenames #######################################################
nameRoot = katdata.obs_params.get('capture_block_id',
katdata.experiment_id)
if type(nameRoot) == list:
nameRoot = nameRoot[0]
fileRoot = os.path.join(outputdir, nameRoot) # root of file name
logFile = fileRoot + ".log" # Processing log file
avgClass = ("UVAv")[0:6] # Averaged data AIPS class
manifestfile = outputdir + '/manifest.pickle'
############################# Initialize OBIT and AIPS ##########################################
noScrat = []
# Logging directly to logFile
OErr.PInit(err, 2, logFile)
EVLAAddOutFile(os.path.basename(logFile), 'project', 'Pipeline log file')
if kwargs.get('reuse'):
ObitSys = AIPSSetup.AIPSSetup(err,
configfile=kwargs.get('configFile'),
scratchdir=kwargs.get('scratchdir'),
aipsdisk=kwargs.get('aipsdisk'),
overwrite=False)
else:
ObitSys = AIPSSetup.AIPSSetup(err,
configfile=kwargs.get('configFile'),
scratchdir=kwargs.get('scratchdir'),
aipsdisk=kwargs.get('aipsdisk'))
# Get the set up AIPS environment.
AIPS_ROOT = os.environ['AIPS_ROOT']
AIPS_VERSION = os.environ['AIPS_VERSION']
nThreads = 72
user = OSystem.PGetAIPSuser()
AIPS.userno = user
disk = 1
fitsdisk = 1
nam = nameRoot[:10]
clss = "Raw"
seq = 1
############### Condition data #########################
#Get calibrator models
fluxcals = katpoint.Catalogue(
open(FITSDir.FITSdisks[1] + "/PERLEY_BUTLER_2013.csv"))
#Condition data (get bpcals, update names for aips conventions etc)
KATh5Condition(katdata, fluxcals, err)
############################# Initialise Parameters ##########################################
####### Initialize parameters dictionary #####
parms = KATInitContParms()
parms['PolCal'] = kwargs.get('polcal')
parms['XYtarg'] = kwargs.get('XYtarg')
# Get default XYtarg if it is not set
targs = [targ.name for targ in katdata.catalogue.targets]
if parms['PolCal']:
if parms['XYtarg'] is None:
GOTTARG = False
for targ in ['1934-638', '0408-65']:
if targ in targs:
parms['XYtarg'] = targ
GOTTARG = True
break
if not GOTTARG:
raise RuntimeError(
'No default targets (1934-638, 0408-65) for XYFix. Cannot run in PolCal mode.'
)
else:
if parms['XYtarg'] not in targs:
raise RuntimeError(
'XYtarg target %s not in observation. Cannot run in PolCal mode.'
% (parms['XYtarg']))
####### User defined parameters ######
if kwargs.get('parmFile'):
print("parmFile", kwargs.get('parmFile'))
exec(open(kwargs.get('parmFile')).read())
EVLAAddOutFile(os.path.basename(kwargs.get('parmFile')), 'project',
'Pipeline input parameters')
###################### Data selection and static edits ############################################
# Select data based on static imageable parameters
KATh5Select(katdata, parms, err, **kwargs)
# General AIPS data parameters at script level
dataClass = ("UVDa")[0:6] # AIPS class of raw uv data
delayClass = "DELA"
band = katdata.spectral_windows[0].product #Correlator product
project = os.path.basename(os.path.splitext(files[0])[0])[
0:10] # Project name (12 char or less, used as AIPS Name)
outIClass = parms["outIClass"] # image AIPS class
debug = parms["debug"]
check = parms["check"]
####################### Import data into AIPS #####################################################
# Reuse or nay?
sw = katdata.spectral_windows[katdata.spw]
# Pick up static flags
if sw.band == 'L':
sflags = FetchObject(ObitTalkUtil.FITSDir.FITSdisks[fitsdisk] +
'maskred.pickle')
if kwargs.get('flag', None):
mess = 'Using static RFI mask in file %s for L-band' % (
ObitTalkUtil.FITSDir.FITSdisks[fitsdisk] + 'maskred.pickle', )
printMess(mess, logFile)
elif sw.band == 'UHF':
sflags = FetchObject(ObitTalkUtil.FITSDir.FITSdisks[fitsdisk] +
'maskredUHF.pickle')
if kwargs.get('flag', None):
mess = 'Using static RFI mask in file %s for UHF-band' % (
ObitTalkUtil.FITSDir.FITSdisks[fitsdisk] +
'maskredUHF.pickle', )
printMess(mess, logFile)
else:
sflags = np.zeros(sw.num_chans, dtype=np.bool)
sflags = sflags[katdata.channels]
# Construct a template uvfits file from master template
mastertemplate = ObitTalkUtil.FITSDir.FITSdisks[
fitsdisk] + 'MKATTemplate.uvtab.gz'
outtemplate = nam + '.uvtemp'
if kwargs.get('reuse'):
uv = UV.newPAUV("AIPS UV DATA", EVLAAIPSName(project), dataClass, disk,
seq, True, err)
obsdata = KATH5toAIPS.GetKATMeta(katdata, err)
# Extract AIPS parameters of the uv data to the metadata
obsdata["Aproject"] = uv.Aname
obsdata["Aclass"] = uv.Aclass
obsdata["Aseq"] = uv.Aseq
obsdata["Adisk"] = disk
obsdata["calInt"] = katdata.dump_period
obsdata["fitsdisk"] = fitsdisk
# TODO: Check if the input data has been Hanned.
doneHann = True
else:
mess = '\nLoading UV data with CBID: %s' % (
katdata.obs_params['capture_block_id'], )
printMess(mess, logFile)
KATH5toAIPS.MakeTemplate(mastertemplate, outtemplate, katdata)
uv = OTObit.uvlod(outtemplate, 0, EVLAAIPSName(project), clss, disk,
seq, err)
obsdata = KATH5toAIPS.KAT2AIPS(katdata,
uv,
disk,
fitsdisk,
err,
calInt=katdata.dump_period,
static=sflags,
**kwargs)
MakeIFs.UVMakeIF(uv, 8, err, solInt=katdata.dump_period)
os.remove(outtemplate)
if parms["PolCal"]:
mess = '\nLoading delay calibration with CBID: %s' % (
delay_katdata.obs_params['capture_block_id'], )
printMess(mess, logFile)
# Load the delay cal observation
KATH5toAIPS.MakeTemplate(mastertemplate, outtemplate, katdata)
delay_uv = OTObit.uvlod(outtemplate, 0, EVLAAIPSName(project),
delayClass, disk, seq, err)
KATH5toAIPS.KAT2AIPS(delay_katdata,
delay_uv,
disk,
fitsdisk,
err,
calInt=katdata.dump_period,
static=sflags,
flag=False)
MakeIFs.UVMakeIF(delay_uv, 8, err, solInt=katdata.dump_period)
os.remove(outtemplate)
# Print the uv data header to screen.
uv.Header(err)
############################# Set Project Processing parameters ###################################
# Parameters derived from obsdata and katdata
KATGetObsParms(obsdata, katdata, parms, logFile)
###### Initialise target parameters #####
KATInitTargParms(katdata, parms, err)
# Load the outputs pickle jar
EVLAFetchOutFiles()
OSystem.PAllowThreads(nThreads) # Allow threads in Obit/oython
retCode = 0
doBand = -1
BPVer = 0
maxgap = max(parms["CalAvgTime"], 160. * katdata.dump_period) / 60.
################### Start processing ###############################################################
mess = "Start project "+parms["project"]+" AIPS user no. "+str(AIPS.userno)+\
", KAT7 configuration "+parms["KAT7Cfg"]
printMess(mess, logFile)
if debug:
pydoc.ttypager = pydoc.plainpager # don't page task input displays
mess = "Using Debug mode "
printMess(mess, logFile)
if check:
mess = "Only checking script"
printMess(mess, logFile)
# Log parameters
printMess("Parameter settings", logFile)
for p in parms:
mess = " " + p + ": " + str(parms[p])
printMess(mess, logFile)
clist = []
for DCal in parms["DCals"]:
if DCal["Source"] not in clist:
clist.append(DCal["Source"])
for PCal in parms["PCals"]:
if PCal["Source"] not in clist:
clist.append(PCal["Source"])
for ACal in parms["ACals"]:
if ACal["Source"] not in clist:
clist.append(ACal["Source"])
if kwargs.get('targets') is not None:
targets = [
targ.name for targ in katdata.catalogue
if (targ.name not in clist) and (
targ.name in kwargs.get('targets').split(','))
]
else:
targets = [
targ.name for targ in katdata.catalogue if (targ.name not in clist)
]
refAnt = kwargs.get('refant')
if refAnt is not None:
try:
SaveObject(obsdata['antLookup'][refAnt],
fileRoot + ".refAnt.pickle", True)
except:
mess = "Select reference antenna " + refAnt + " not in antenna table."
printMess(mess, logFile)
print(mess)
refAnt = FetchObject(fileRoot + ".refAnt.pickle")
# Save parameters to pickle jar, manifest
ParmsPicklefile = fileRoot + ".Parms.pickle" # Where results saved
SaveObject(parms, ParmsPicklefile, True)
EVLAAddOutFile(os.path.basename(ParmsPicklefile), 'project',
'Processing parameters used')
loadClass = dataClass
# Hanning - only if not reusing
doneHann = False
if not kwargs.get('reuse'):
if parms["doHann"]:
uv = KATHann(uv, EVLAAIPSName(project), dataClass, disk, seq, err, \
doDescm=parms["doDescm"], flagVer=-1, logfile=logFile, zapin=True, check=check, debug=debug)
doneHann = True
if parms["PolCal"] and parms["doHann"]:
mess = "Hanning delay calibration scan"
printMess(mess, logFile)
delay_uv = KATHann(delay_uv, EVLAAIPSName(project), delayClass, disk, seq + 1, err, \
doDescm=parms["doDescm"], flagVer=-1, logfile=logFile, zapin=True, check=check, debug=debug)
if doneHann:
# Halve channels after hanning.
parms["selChan"] = int(parms["selChan"] / 2)
parms["BChDrop"] = int(parms["BChDrop"] / 2)
parms["EChDrop"] = int(parms["EChDrop"] / 2)
if uv == None and not check:
raise RuntimeError("Cannot Hann data ")
# Clear any old calibration/editing
if parms["doClearTab"] or kwargs.get('reuse'):
mess = "Clear previous calibration"
printMess(mess, logFile)
EVLAClearCal(uv,
err,
doGain=parms["doClearGain"],
doFlag=parms["doClearFlag"],
doBP=parms["doClearBP"],
check=check)
OErr.printErrMsg(err, "Error resetting calibration")
# Copy FG 1 to FG 2
if parms["doCopyFG"]:
mess = "Copy FG 1 to FG 2"
printMess(mess, logFile)
retCode = KATCopyFG(uv, err, logfile=logFile, check=check, debug=debug)
if retCode != 0:
raise RuntimeError("Error Copying FG table")
# Flag antennas shadowed by others?
if parms["doShad"]:
retCode = EVLAShadow (uv, err, shadBl=parms["shadBl"], \
logfile=logFile, check=check, debug=debug)
if retCode != 0:
raise RuntimeError("Error Shadow flagging data")
# Median window time editing, for RFI impulsive in time
if parms["doMednTD1"]:
mess = "Median window time editing, for RFI impulsive in time:"
printMess(mess, logFile)
retCode = EVLAMedianFlag (uv, clist, err, noScrat=noScrat, nThreads=nThreads, \
avgTime=parms["mednAvgTime"], avgFreq=parms["mednAvgFreq"], chAvg= parms["mednChAvg"], \
timeWind=parms["mednTimeWind"],flagVer=2, flagTab=2,flagSig=parms["mednSigma"], \
logfile=logFile, check=check, debug=False)
if retCode != 0:
raise RuntimeError("Error in MednFlag")
# Median window frequency editing, for RFI impulsive in frequency
if parms["doFD1"]:
mess = "Median window frequency editing, for RFI impulsive in frequency:"
printMess(mess, logFile)
retCode = EVLAAutoFlag (uv, clist, err, flagVer=2, flagTab=2, doCalib=-1, doBand=-1, \
timeAvg=parms["FD1TimeAvg"], \
doFD=True, FDmaxAmp=1.0e20, FDmaxV=1.0e20, FDwidMW=parms["FD1widMW"], \
FDmaxRMS=[1.0e20,0.1], FDmaxRes=parms["FD1maxRes"], \
FDmaxResBL= parms["FD1maxRes"], FDbaseSel=parms["FD1baseSel"],\
nThreads=nThreads, logfile=logFile, check=check, debug=debug)
if retCode != 0:
raise RuntimeError("Error in AutoFlag")
# Parallactic angle correction?
if parms["doPACor"]:
retCode = EVLAPACor(uv, err, \
logfile=logFile, check=check, debug=debug)
if retCode != 0:
raise RuntimeError("Error in Parallactic angle correction")
# Need to find a reference antenna? See if we have saved it?
if (parms["refAnt"] <= 0):
refAnt = FetchObject(fileRoot + ".refAnt.pickle")
if refAnt:
parms["refAnt"] = refAnt
# Use bandpass calibrator and center half of each spectrum
if parms["refAnt"] <= 0:
mess = "Find best reference antenna: run Calib on BP Cal(s) "
printMess(mess, logFile)
parms["refAnt"] = EVLAGetRefAnt(uv, parms["BPCals"], err, flagVer=0, \
solInt=parms["bpsolint1"], nThreads=nThreads, \
logfile=logFile, check=check, debug=debug)
if err.isErr:
raise RuntimeError("Error finding reference antenna")
if parms["refAnts"][0] <= 0:
parms["refAnts"][0] = parms["refAnt"]
mess = "Picked reference antenna " + str(parms["refAnt"])
printMess(mess, logFile)
# Save it
ParmsPicklefile = fileRoot + ".Parms.pickle" # Where results saved
SaveObject(parms, ParmsPicklefile, True)
refAntPicklefile = fileRoot + ".refAnt.pickle" # Where results saved
SaveObject(parms["refAnt"], refAntPicklefile, True)
# Plot Raw, edited data?
if parms["doRawSpecPlot"] and parms["plotSource"]:
mess = "Raw Spectral plot for: " + ' '.join(parms["BPCal"])
printMess(mess, logFile)
plotFile = fileRoot + "_RawSpec.ps"
retCode = EVLASpectrum(uv, parms["BPCal"], parms["plotTime"], maxgap, plotFile, parms["refAnt"], err, \
Stokes=["RR","LL"], doband=-1, \
check=check, debug=debug, logfile=logFile )
if retCode != 0:
raise RuntimeError("Error in Plotting spectrum")
EVLAAddOutFile(plotFile, 'project', 'Pipeline log file')
if parms["PolCal"]:
mess = "XYphase bandpass calibration"
printMess(mess, logFile)
retCode = KATXPhase(delay_uv,
uv,
err,
logfile=logFile,
check=check,
debug=debug,
doCalib=-1,
flagVer=0,
doBand=-1,
refAnt=parms['refAnt'])
doBand = 1
BPVer += 1
if retCode != 0:
raise RuntimeError("Error in Xphase calibration")
# delay calibration
if parms["doDelayCal"] and parms["DCals"] and not check:
plotFile = fileRoot + "_DelayCal.ps"
retCode = EVLADelayCal(uv, parms["DCals"], err, \
BChan=parms["delayBChan"], EChan=parms["delayEChan"], \
doCalib=-1, flagVer=0, doBand=doBand, BPVer=BPVer, \
solInt=parms["delaySolInt"], smoTime=parms["delaySmoo"], \
refAnts=[parms["refAnt"]], doTwo=parms["doTwo"],
doZeroPhs=parms["delayZeroPhs"], \
doAvgIF=parms["delayAvgIF"], doAvgPol=parms["delayAvgPol"], \
doPlot=parms["doSNPlot"], plotFile=plotFile, \
nThreads=nThreads, noScrat=noScrat, \
logfile=logFile, check=check, debug=debug)
if retCode != 0:
raise RuntimeError("Error in delay calibration")
# Plot corrected data?
if parms["doSpecPlot"] and parms["plotSource"]:
plotFile = fileRoot + "_DelaySpec.ps"
retCode = EVLASpectrum(uv, parms["BPCal"], parms["plotTime"], maxgap, \
plotFile, parms["refAnt"], err, \
Stokes=["RR","LL"], doband=doBand, \
check=check, debug=debug, logfile=logFile )
if retCode != 0:
raise RuntimeError("Error in Plotting spectrum")
# Bandpass calibration
if parms["doBPCal"] and parms["BPCals"]:
retCode = KATBPCal(uv, parms["BPCals"], err, doBand=doBand, BPVer=BPVer, newBPVer=0,
noScrat=noScrat, solInt1=parms["bpsolint1"], \
solInt2=parms["bpsolint2"], solMode=parms["bpsolMode"], \
BChan1=parms["bpBChan1"], EChan1=parms["bpEChan1"], \
BChan2=parms["bpBChan2"], EChan2=parms["bpEChan2"], ChWid2=parms["bpChWid2"], \
doCenter1=parms["bpDoCenter1"], refAnt=parms["refAnt"], \
UVRange=parms["bpUVRange"], doCalib=2, gainUse=0, flagVer=0, doPlot=False, \
nThreads=nThreads, logfile=logFile, check=check, debug=debug)
if retCode != 0:
raise RuntimeError("Error in Bandpass calibration")
# Plot corrected data?
if parms["doSpecPlot"] and parms["plotSource"]:
plotFile = fileRoot + "_BPSpec.ps"
retCode = EVLASpectrum(uv, parms["BPCal"], parms["plotTime"], maxgap, plotFile, \
parms["refAnt"], err, Stokes=["RR","LL"], doband=1, \
check=check, debug=debug, logfile=logFile )
if retCode != 0:
raise RuntimeError("Error in Plotting spectrum")
# Amp & phase Calibrate
if parms["doAmpPhaseCal"]:
plotFile = fileRoot + "_APCal.ps"
retCode = KATCalAP (uv, [], parms["ACals"], err, PCals=parms["PCals"],
doCalib=2, doBand=1, BPVer=0, flagVer=0, \
BChan=parms["ampBChan"], EChan=parms["ampEChan"], \
solInt=parms["solInt"], solSmo=parms["solSmo"], ampScalar=parms["ampScalar"], \
doAmpEdit=parms["doAmpEdit"], ampSigma=parms["ampSigma"], \
ampEditFG=parms["ampEditFG"], avgPol=parms["PolCal"], \
doPlot=parms["doSNPlot"], plotFile=plotFile, refAnt=parms["refAnt"], \
nThreads=nThreads, noScrat=noScrat, logfile=logFile, check=check, debug=debug)
if retCode != 0:
raise RuntimeError("Error calibrating")
# More editing
if parms["doAutoFlag"]:
mess = "Post calibration editing:"
printMess(mess, logFile)
# if going to redo then only calibrators
if parms["doRecal"]:
# Only calibrators
clist = []
for DCal in parms["DCals"]:
if DCal["Source"] not in clist:
clist.append(DCal["Source"])
for PCal in parms["PCals"]:
if PCal["Source"] not in clist:
clist.append(PCal["Source"])
for ACal in parms["ACals"]:
if ACal["Source"] not in clist:
clist.append(ACal["Source"])
else:
clist = []
retCode = EVLAAutoFlag (uv, clist, err, flagVer=0, flagTab =2, \
doCalib=2, gainUse=0, doBand=1, BPVer=BPVer, \
IClip=parms["IClip"], minAmp=parms["minAmp"], timeAvg=parms["timeAvg"], \
doFD=parms["doFirstAFFD"], FDmaxAmp=parms["FDmaxAmp"], FDmaxV=parms["FDmaxV"], \
FDwidMW=parms["FDwidMW"], FDmaxRMS=parms["FDmaxRMS"], \
FDmaxRes=parms["FDmaxRes"], FDmaxResBL=parms["FDmaxResBL"], \
FDbaseSel=parms["FDbaseSel"], \
nThreads=nThreads, logfile=logFile, check=check, debug=debug)
if retCode != 0:
raise RuntimeError("Error in AutoFlag")
# Redo the calibration using new flagging?
if parms["doBPCal2"] == None:
parms["doBPCal2"] = parms["doBPCal"]
if parms["doDelayCal2"] == None:
parms["doDelayCal2"] = parms["doDelayCal2"]
if parms["doAmpPhaseCal2"] == None:
parms["doAmpPhaseCal2"] = parms["doAmpPhaseCal"]
if parms["doAutoFlag2"] == None:
parms["doAutoFlagCal2"] = parms["doAutoFlag"]
if parms["doRecal"]:
mess = "Redo calibration:"
printMess(mess, logFile)
EVLAClearCal(uv,
err,
doGain=True,
doFlag=False,
doBP=True,
check=check,
logfile=logFile)
OErr.printErrMsg(err, "Error resetting calibration")
BPVer = 0
# Parallactic angle correction?
if parms["doPACor"]:
retCode = EVLAPACor(uv, err, \
logfile=logFile, check=check, debug=debug)
if retCode != 0:
raise RuntimeError("Error in Parallactic angle correction")
# Run MKXPhase on delaycal data and attach BP table to UV data
if parms["PolCal"]:
mess = "XYphase bandpass calibration"
printMess(mess, logFile)
retCode = KATXPhase(delay_uv,
uv,
err,
logfile=logFile,
check=check,
debug=debug,
doCalib=-1,
flagVer=0,
doBand=-1,
refAnt=parms['refAnt'])
BPVer += 1
if retCode != 0:
raise RuntimeError("Error in Xphase calibration")
# Delay recalibration
if parms["doDelayCal2"] and parms["DCals"] and not check:
plotFile = fileRoot + "_DelayCal2.ps"
retCode = EVLADelayCal(uv, parms["DCals"], err, \
BChan=parms["delayBChan"], EChan=parms["delayEChan"], \
doCalib=-1, flagVer=0, doBand=doBand, BPVer=BPVer, \
solInt=parms["delaySolInt"], smoTime=parms["delaySmoo"], \
refAnts=[parms["refAnt"]], doTwo=parms["doTwo"], \
doZeroPhs=parms["delayZeroPhs"], \
doAvgIF=parms["delayAvgIF"], doAvgPol=parms["delayAvgPol"], \
doPlot=parms["doSNPlot"], plotFile=plotFile, \
nThreads=nThreads, noScrat=noScrat, \
logfile=logFile, check=check, debug=debug)
if retCode != 0:
raise RuntimeError("Error in delay calibration")
# Plot corrected data?
if parms["doSpecPlot"] and parms["plotSource"]:
plotFile = fileRoot + "_DelaySpec2.ps"
retCode = EVLASpectrum(uv, parms["BPCal"], parms["plotTime"], maxgap, plotFile, parms["refAnt"], err, \
Stokes=["RR","LL"], doband=doband, \
check=check, debug=debug, logfile=logFile )
if retCode != 0:
raise RuntimeError("Error in Plotting spectrum")
# Bandpass calibration
if parms["doBPCal2"] and parms["BPCals"]:
retCode = KATBPCal(uv, parms["BPCals"], err, doBand=doBand, BPVer=BPVer, newBPVer=0, \
noScrat=noScrat, solInt1=parms["bpsolint1"], \
solInt2=parms["bpsolint2"], solMode=parms["bpsolMode"], \
BChan1=parms["bpBChan1"], EChan1=parms["bpEChan1"], \
BChan2=parms["bpBChan2"], EChan2=parms["bpEChan2"], ChWid2=parms["bpChWid2"], \
doCenter1=parms["bpDoCenter1"], refAnt=parms["refAnt"], \
UVRange=parms["bpUVRange"], doCalib=2, gainUse=0, flagVer=0, doPlot=False, \
nThreads=nThreads, logfile=logFile, check=check, debug=debug)
if retCode != 0:
raise RuntimeError("Error in Bandpass calibration")
# Plot corrected data?
if parms["doSpecPlot"] and parms["plotSource"]:
plotFile = fileRoot + "_BPSpec2.ps"
retCode = EVLASpectrum(uv, parms["BPCal"], parms["plotTime"], maxgap, plotFile, parms["refAnt"], err, \
Stokes=["RR","LL"], doband=1, \
check=check, debug=debug, logfile=logFile )
if retCode != 0:
raise RuntimeError("Error in Plotting spectrum")
# Amp & phase Recalibrate
if parms["doAmpPhaseCal2"]:
plotFile = fileRoot + "_APCal2.ps"
retCode = KATCalAP (uv, [], parms["ACals"], err, PCals=parms["PCals"], \
doCalib=2, doBand=1, BPVer=0, flagVer=0, \
BChan=parms["ampBChan"], EChan=parms["ampEChan"], \
solInt=parms["solInt"], solSmo=parms["solSmo"], ampScalar=parms["ampScalar"], \
doAmpEdit=True, ampSigma=parms["ampSigma"], \
ampEditFG=parms["ampEditFG"], avgPol=parms["PolCal"], \
doPlot=parms["doSNPlot"], plotFile=plotFile, refAnt=parms["refAnt"], \
noScrat=noScrat, nThreads=nThreads, logfile=logFile, check=check, debug=debug)
if retCode != 0:
raise RuntimeError("Error calibrating")
# More editing
if parms["doAutoFlag2"]:
mess = "Post recalibration editing:"
printMess(mess, logFile)
retCode = EVLAAutoFlag (uv, [], err, flagVer=0, flagTab=2, \
doCalib=2, gainUse=0, doBand=1, BPVer=0, \
IClip=parms["IClip"], minAmp=parms["minAmp"], timeAvg=parms["timeAvg"], \
doFD=parms["doSecAFFD"], FDmaxAmp=parms["FDmaxAmp"], FDmaxV=parms["FDmaxV"], \
FDwidMW=parms["FDwidMW"], FDmaxRMS=parms["FDmaxRMS"], \
FDmaxRes=parms["FDmaxRes"], FDmaxResBL= parms["FDmaxResBL"], \
FDbaseSel=parms["FDbaseSel"], \
nThreads=nThreads, logfile=logFile, check=check, debug=debug)
if retCode != 0:
raise RuntimeError("Error in AutoFlag")
# end recal
# Calibrate and average data
# Overwrite avgStokes from command line
if kwargs.get('halfstokes'):
parms["avgStokes"] = 'HALF'
if parms["doCalAvg"] == 'Splat':
retCode = KATCalAvg (uv, avgClass, parms["seq"], parms["CalAvgTime"], err, \
flagVer=2, doCalib=2, gainUse=0, doBand=1, BPVer=0, doPol=False, \
avgFreq=parms["avgFreq"], chAvg=parms["chAvg"], Stokes=parms["avgStokes"], \
BChan=1, EChan=parms["selChan"] - 1, doAuto=parms["doAuto"], \
BIF=parms["CABIF"], EIF=parms["CAEIF"], Compress=parms["Compress"], \
nThreads=nThreads, logfile=logFile, check=check, debug=debug)
if retCode != 0:
raise RuntimeError("Error in CalAvg")
elif parms["doCalAvg"] == 'BL':
retCode = KATBLCalAvg (uv, avgClass, parms["seq"], err, \
flagVer=2, doCalib=2, gainUse=0, doBand=1, BPVer=0, doPol=False, \
avgFreq=parms["avgFreq"], chAvg=parms["chAvg"], FOV=parms['FOV'], \
maxInt=min(parms["solPInt"],parms["solAInt"]), Stokes=parms["avgStokes"], \
BChan=1, EChan=parms["selChan"] - 1, timeAvg=parms["CalAvgTime"], \
BIF=parms["CABIF"], EIF=parms["CAEIF"], Compress=parms["Compress"], \
logfile=logFile, check=check, debug=debug)
if retCode != 0:
raise RuntimeError("Error in BLCalAvg")
if parms["doSaveTab"]:
filename = project + ".CalTab.uvtab"
_ = EVLAUVFITSTab(uv, filename, 0, err, logfile=logFile)
#Zap unaveraged data if requested
if kwargs.get('zapraw'):
uv.Zap(err)
# Get calibrated/averaged data
if not check:
uv = UV.newPAUV("AIPS UV DATA", EVLAAIPSName(project), avgClass[0:6], \
disk, parms["seq"], True, err)
if err.isErr:
OErr.printErrMsg(err, "Error creating cal/avg AIPS data")
plotFile = fileRoot + "_Spec.ps"
retCode = EVLASpectrum(uv, parms["BPCal"], parms["plotTime"], maxgap, \
plotFile, parms["refAnt"], err, \
Stokes=["I"], doband=-1, docalib=-1, \
check=check, debug=debug, logfile=logFile )
if retCode != 0:
raise RuntimeError("Error in Plotting spectrum")
# KATUVFITS(uv, 'preimage.uvfits', 0, err, exclude=["AIPS HI", "AIPS SL", "AIPS PL"],
# include=["AIPS AN", "AIPS FQ"], compress=parms["Compress"], logfile=logFile)
KATUVFITab(uv, project + '.uvtab', 0, err)
#Gzip the data?
if kwargs.get('gzip'):
os.system('pigz -p %d %s' % (nThreads, project + '.uvtab'))
os.system('rm -f %s' % (project + '.uvtab'))
def open_uv(aips_path, nvispio=1024, mode=None):
"""
Opens an AIPS/FITS UV file and returns a wrapped :class:`UVFacade` object.
Parameters
----------
aips_path: :class:`AIPSPath`
Obit file object.
nvispio: integer
Number of visibilities to read/write per I/O operation
mode(optional): str
"r" to read, "w" to write, "rw" to read and write.
Defaults to "r"
Returns
-------
:class:`UV`
An Obit UV object
"""
err = obit_err()
if mode is None:
mode = "r"
uv_mode = uv_file_mode(mode)
exists = False # Test if the file exists
if aips_path.dtype == "AIPS":
try:
uv = UV.newPAUV(aips_path.label,
aips_path.name,
aips_path.aclass,
aips_path.disk,
aips_path.seq,
exists,
err,
nvis=nvispio)
except Exception:
raise ValueError("Error calling newPAUV on '%s'" % aips_path)
elif aips_path.dtype == "FITS":
raise NotImplementedError("newPFUV calls do not currently work")
try:
uv = UV.newPFUV(aips_path.label,
aips_path.name,
aips_path.disk,
exists,
err,
nvis=nvispio)
except Exception:
raise ValueError("Error calling newPFUV on '%s'" % aips_path)
else:
raise ValueError("Invalid dtype '{}'".format(aips_path.dtype))
handle_obit_err("Error opening '%s'" % aips_path, err)
try:
uv.Open(uv_mode, err)
except Exception:
raise ValueError("Error opening '%s'" % aips_path)
handle_obit_err("Error opening '%s'" % aips_path, err)
return uv