def UVMakeIF(outUV, nIF, err, solInt=10.):
"""
Change number of IFs from 1 to nIF
Operation done in place
* outUV = output Obit UV object
Only really works for AIPS
* nIF = number of desired output IFs
MUST be the same number of channels per IF
* err = Obit error/message stack
* solInt = Solution interval for remade CL table.
"""
################################################################
# Checks
if not UV.PIsA(outUV):
raise TypeError("outUV MUST be a defined Python Obit UV")
# Input can have 1 or no IFs defined
jlocif = outUV.Desc.Dict["jlocif"]
if jlocif >= 0 and outUV.Desc.Dict["inaxes"][jlocif] > 1:
raise RuntimeError("Input UV has excessive IFs already:" \
+str(outUV.Desc.Dict["inaxes"][jlocif]))
# Check number of requested IFs
if nIF < 2:
raise RuntimeError("Too few output IFs requested: " + str(nIF))
# Must be an even number of channels per output IF
jlocf = outUV.Desc.Dict["jlocf"]
nchan = outUV.Desc.Dict["inaxes"][jlocf]
if (nchan % nIF) != 0:
raise RuntimeError("Unequal numbers of channels per " + str(nIF) +
" IFs")
# Patch UV Descriptor
DescMakeIF(outUV, nIF, err)
# Convert FQ Table
UpdateFQ2(outUV, nIF, err)
# Convert AN Table
maxant = UpdateAN2(outUV, nIF, err)
# Convert SU Table
UpdateSU2(outUV, nIF, err)
# Regenerate CL table 1 - delete any old
outUV.ZapTable("AIPS CL", -1, err)
print('(Re) generate CL table')
UV.PTableCLfromNX(outUV, maxant, err, calInt=solInt)
# dummy FG 1
print('Dummy entry in Flag table 1')
UV.PFlag(outUV,
err,
timeRange=[-10., -9.],
Ants=[200, 200],
Stokes='0000',
Reason='Dummy')
# Update
outUV.UpdateDesc(err)
OErr.printErrMsg(err, "Error updating output")
def WriteFGTable(outUV, katdata, meta, err):
"""
Get the flags from the h5 file and convert to FG table format.
UNUSED- This implimentation is too slow!
outUV = Obit UV object
meta = dict with data meta data
err = Python Obit Error/message stack to init
"""
###############################################################
# work out Start time in unix sec
tm = katdata.timestamps[1:2]
tx = time.gmtime(tm[0])
time0 = tm[0] - tx[3] * 3600.0 - tx[4] * 60.0 - tx[5]
int_time = katdata.dump_period
#Loop through scans in h5 file
row = 0
for scan, state, target in katdata.scans():
name = target.name.replace(' ', '_')
if state != 'track':
continue
tm = katdata.timestamps[:]
nint = len(tm)
el = target.azel(tm[int(nint / 2)])[1] * 180. / math.pi
if el < 15.0:
continue
row += 1
flags = katdata.flags()[:]
numflag = 0
for t, chan_corr in enumerate(flags):
for c, chan in enumerate(chan_corr):
cpflagged = []
for p, cp in enumerate(chan):
#for cpaverage in meta['pairLookup']:
flag = cp #numpy.any(chan[meta['pairLookup'][cpaverage]])
product = meta['products'][p]
if product[0] == product[1]:
continue
if flag and (not product[0:2] in cpflagged):
cpflagged.append(product[0:2])
numflag += 1
starttime = float(
(tm[t] - time0 - (int_time / 2)) / 86400.0)
endtime = float(
(tm[t] - time0 + (int_time / 2)) / 86400.0)
UV.PFlag(outUV,err,timeRange=[starttime,endtime], flagVer=1, \
Ants=[product[0],product[1]], \
Chans=[c+1,c+1], IFs=[1,1], Stokes='1111', Reason='Online flag')
numvis = t * c * (p / meta["nstokes"])
msg = "Scan %4d %16s Online flags: %7s of %8s vis" % (
row, name, numflag, numvis)
OErr.PLog(err, OErr.Info, msg)
OErr.printErr(err)
def pipeline(scriptName, aipsSetup, parmFile):
"""
Linear (feed) Polarization Continuum pipeline.
* *scriptName* = this script file name
* *aipsSetup* = AIPS setup file
* *parmFile* = pipeline input parameters file
"""
############################# Initialize OBIT ##########################################
noScrat = []
exec(open(aipsSetup).read())
EVLAAddOutFile(aipsSetup, 'project', "Obit's AIPS setup file")
############################# Default parameters ##########################################
# Initialize parameters
parms = EVLAInitContParms()
# Linear feed specific parameters
# Relative gain calibration
parms["doXYRelGain"] = False # constrain relative X, Y gain amplitudes
parms[
"doXYRelGain2"] = None # constrain relative X, Y gain amplitudes, 2nd pass
parms["XYRelGainCal"] = None # X/Y gain calibrator structure
parms["XYRelGainCalTime"] = [0., 0.
] # Time range for relative gain calibration
############################# Set Project Processing parameters ##################
print("parmFile", parmFile)
exec(open(parmFile).read())
EVLAAddOutFile(parmFile, 'project', 'Pipeline input parameters')
# frequency/configuration dependent default parameters
EVLAInitContFQParms(parms)
# Linear feed specific:
parms["avgPol"] = parms["doXYRelGain"] and (parms["XYRelGainCal"] != None
) # Fix relative X/Y gains?
# General data parameters
band = parms["band"] # Observing band
dataClass = ("UVDa" + band)[0:6] # AIPS class of raw uv data
project = parms["project"][
0:12] # Project name (12 char or less, used as AIPS Name)
session = parms["session"] # Project session code
################################## Process #####################################
fileRoot = parms["project"] + "_" + parms["session"] + "_" + parms[
"band"] # root of file name
logFile = fileRoot + ".log" # Processing log file
uv = None
uvc = None
avgClass = ("UVAv" + band)[0:6] # Averaged data AIPS class
outIClass = parms["outIClass"] # image AIPS class
# Load the outputs pickle jar
EVLAFetchOutFiles()
# Logging directly to logFile
OErr.PInit(err, parms["prtLv"], logFile)
OSystem.PAllowThreads(nThreads) # Allow threads in Obit/oython
retCode = 0
EVLAAddOutFile(logFile, 'project', 'Pipeline log file')
mess = "Start project "+parms["project"]+" session "+parms["session"]+\
" "+parms["band"]+" Band"+" AIPS user no. "+str(AIPS.userno)+\
", EVLA configuration "+parms["VLACfg"]
printMess(mess, logFile)
if debug:
pydoc.ttypager = pydoc.plainpager # don't page task input displays
mess = "Using Debug mode "
printMess(mess, logFile)
if check:
mess = "Only checking script"
printMess(mess, logFile)
# Log parameters
printMess("Parameter settings", logFile)
for p in parms:
mess = " " + p + ": " + str(parms[p])
printMess(mess, logFile)
# Save parameters to pickle jar, manifest
ParmsPicklefile = project + "_" + session + "_" + band + ".Parms.pickle" # Where results saved
SaveObject(parms, ParmsPicklefile, True)
EVLAAddOutFile(ParmsPicklefile, 'project', 'Processing parameters used')
# Are we going to be doing Hanning?
if parms["doHann"]:
loadClass = parms["band"] + "Raw"
else:
loadClass = dataClass
# Load Data from Archive directory
if parms["doLoadArchive"]:
uv = EVLAUVLoadArch(parms["archRoot"], EVLAAIPSName(project, session), loadClass, disk, parms["seq"], err, \
selConfig=parms["selConfig"], doSwPwr=parms["doSwPwr"], \
selBand=parms["selBand"], selChan=parms["selChan"], \
selNIF=parms["selNIF"], calInt=parms["calInt"], \
logfile=logFile, Compress=parms["Compress"], check=check, debug=debug)
if uv == None and not check:
raise RuntimeError("Cannot load " + parms["DataRoot"])
# Hanning
if parms["doHann"]:
# Set uv if not done
if uv == None and not check:
uv = UV.newPAUV("AIPS UV DATA", EVLAAIPSName(project, session),
loadClass[0:6], disk, parms["seq"], True, err)
if err.isErr:
OErr.printErrMsg(err, "Error creating AIPS data")
uv = EVLAHann(uv, EVLAAIPSName(project, session), dataClass, disk, parms["seq"], err, \
doDescm=parms["doDescm"], logfile=logFile, check=check, debug=debug)
if uv == None and not check:
raise RuntimeError("Cannot Hann data ")
# Set uv if not done
if uv == None and not check:
uv = UV.newPAUV("AIPS UV DATA", EVLAAIPSName(project, session), dataClass[0:6], \
disk, parms["seq"], True, err)
if err.isErr:
OErr.printErrMsg(err, "Error creating AIPS data")
# Save file names in history
EVLAScriptHistory(uv, scriptName, aipsSetup, parmFile, err)
if err.isErr:
OErr.printErrMsg(err, "Error writing file names to history")
# Clear any old calibration/editing
if parms["doClearTab"]:
mess = "Clear previous calibration"
printMess(mess, logFile)
EVLAClearCal(uv,
err,
doGain=parms["doClearGain"],
doFlag=parms["doClearFlag"],
doBP=parms["doClearBP"],
check=check)
OErr.printErrMsg(err, "Error resetting calibration")
# Copy FG 1 to FG 2
if parms["doCopyFG"]:
mess = "Copy FG 1 to FG 2"
printMess(mess, logFile)
retCode = EVLACopyFG(uv,
err,
logfile=logFile,
check=check,
debug=debug)
if retCode != 0:
raise RuntimeError("Error Copying FG table")
# Drop end channels of spectra? Only if new FG 2
if parms["doCopyFG"] and (parms["BChDrop"] > 0) or (parms["EChDrop"] > 0):
# Channels based on original number, reduced if Hanning
nchan = uv.Desc.Dict["inaxes"][uv.Desc.Dict["jlocf"]]
fact = parms["selChan"] / nchan # Hanning reduction factor
BChDrop = parms["BChDrop"] / fact
EChDrop = parms["EChDrop"] / fact
mess = "Trim %d channels from start and %d from end of each spectrum" % (
BChDrop, EChDrop)
printMess(mess, logFile)
retCode = EVLADropChan (uv, BChDrop, EChDrop, err, flagVer=parms["editFG"], \
logfile=logFile, check=check, debug=debug)
if retCode != 0:
raise RuntimeError("Error Copying FG table")
# Special editing
if parms["doEditList"] and not check:
mess = "Special editing"
printMess(mess, logFile)
for edt in parms["editList"]:
UV.PFlag(uv,err,timeRange=[dhms2day(edt["timer"][0]),dhms2day(edt["timer"][1])], \
flagVer=parms["editFG"], Ants=edt["Ant"], Chans=edt["Chans"], IFs=edt["IFs"], \
Stokes=edt["Stokes"], Reason=edt["Reason"])
OErr.printErrMsg(err, "Error Flagging")
# Quack to remove data from start and end of each scan
if parms["doQuack"]:
retCode = EVLAQuack (uv, err, begDrop=parms["quackBegDrop"], endDrop=parms["quackEndDrop"], \
Reason=parms["quackReason"], \
logfile=logFile, check=check, debug=debug)
if retCode != 0:
raise RuntimeError("Error Quacking data")
# Flag antennas shadowed by others?
if parms["doShad"]:
retCode = EVLAShadow (uv, err, shadBl=parms["shadBl"], \
logfile=logFile, check=check, debug=debug)
if retCode != 0:
raise RuntimeError("Error Shadow flagging data")
# Median window time editing, for RFI impulsive in time
if parms["doMedn"]:
mess = "Median window time editing, for RFI impulsive in time:"
printMess(mess, logFile)
retCode = EVLAMedianFlag (uv, " ", err, noScrat=noScrat, nThreads=nThreads, \
avgTime=parms["avgTime"], avgFreq=parms["avgFreq"], chAvg= parms["chAvg"], \
timeWind=parms["timeWind"], flagVer=2,flagSig=parms["mednSigma"], \
logfile=logFile, check=check, debug=False)
if retCode != 0:
raise RuntimeError("Error in MednFlag")
# Median window frequency editing, for RFI impulsive in frequency
if parms["doFD1"]:
mess = "Median window frequency editing, for RFI impulsive in frequency:"
printMess(mess, logFile)
retCode = EVLAAutoFlag (uv, " ", err, flagVer=2, doCalib=-1, doBand=-1, \
timeAvg=parms["FD1TimeAvg"], \
doFD=True, FDmaxAmp=1.0e20, FDmaxV=1.0e20, FDwidMW=parms["FD1widMW"], \
FDmaxRMS=[1.0e20,0.1], FDmaxRes=parms["FD1maxRes"], \
FDmaxResBL= parms["FD1maxRes"], \
nThreads=nThreads, logfile=logFile, check=check, debug=debug)
if retCode != 0:
raise RuntimeError("Error in AutoFlag")
# RMS/Mean editing for calibrators
if parms["doRMSAvg"]:
mess = "RMS/Mean editing for calibrators:"
printMess(mess, logFile)
clist = [] # Calibrator list
for s in parms["ACals"]:
if s['Source'] not in clist:
clist.append(s['Source'])
for s in parms["PCals"]:
if s['Source'] not in clist:
clist.append(s['Source'])
for s in parms["DCals"]:
if s['Source'] not in clist:
clist.append(s['Source'])
retCode = EVLAAutoFlag (uv, clist, err, flagVer=2, doCalib=-1, doBand=-1, \
RMSAvg=parms["RMSAvg"], timeAvg=parms["RMSTimeAvg"], \
nThreads=nThreads, logfile=logFile, check=check, debug=debug)
if retCode != 0:
raise RuntimeError("Error in AutoFlag")
# Need to find a reference antenna? See if we have saved it?
if (parms["refAnt"] <= 0):
refAnt = FetchObject(project + "_" + session + "_" + band +
".refAnt.pickle")
if refAnt:
parms["refAnt"] = refAnt
# Use bandpass calibrator and center half of each spectrum
if parms["refAnt"] <= 0:
mess = "Find best reference antenna: run Calib on BP Cal(s) "
printMess(mess, logFile)
parms["refAnt"] = EVLAGetRefAnt(uv, parms["BPCals"], err, flagVer=2, \
solInt=parms["bpsolint1"], nThreads=nThreads, \
logfile=logFile, check=check, debug=debug)
if err.isErr:
raise RuntimeError("Error finding reference antenna")
if parms["refAnts"][0] <= 0:
parms["refAnts"][0] = parms["refAnt"]
mess = "Picked reference antenna " + str(parms["refAnt"])
printMess(mess, logFile)
# Save it
ParmsPicklefile = project + "_" + session + "_" + band + ".Parms.pickle" # Where results saved
SaveObject(parms, ParmsPicklefile, True)
refAntPicklefile = project + "_" + session + "_" + band + ".refAnt.pickle" # Where results saved
SaveObject(parms["refAnt"], refAntPicklefile, True)
# Plot Raw, edited data?
if parms["doRawSpecPlot"] and parms["plotSource"]:
mess = "Raw Spectral plot for: " + parms["plotSource"]
printMess(mess, logFile)
plotFile = "./" + fileRoot + "RawSpec.ps"
retCode = EVLASpectrum(uv, parms["plotSource"], parms["plotTime"], plotFile, parms["refAnt"], err, \
Stokes=["RR","LL"], doband=-1, \
check=check, debug=debug, logfile=logFile )
if retCode != 0:
raise RuntimeError("Error in Plotting spectrum")
EVLAAddOutFile(plotFile, 'project', 'Pipeline log file')
# delay calibration
if parms["doDelayCal"] and parms["DCals"] and not check:
plotFile = "./" + fileRoot + "DelayCal.ps"
retCode = EVLADelayCal(uv, parms["DCals"], err, \
BChan=parms["delayBChan"], EChan=parms["delayEChan"], \
doCalib=2, flagVer=2, doBand=-1, \
solInt=parms["delaySolInt"], smoTime=1.0/60.0, \
refAnts=[parms["refAnt"]], doTwo=parms["doTwo"],
doZeroPhs=parms["delayZeroPhs"], \
doPlot=parms["doSNPlot"], plotFile=plotFile, \
nThreads=nThreads, noScrat=noScrat, \
logfile=logFile, check=check, debug=debug)
if retCode != 0:
raise RuntimeError("Error in delay calibration")
# Plot corrected data?
if parms["doSpecPlot"] and parms["plotSource"]:
plotFile = "./" + fileRoot + "DelaySpec.ps"
retCode = EVLASpectrum(uv, parms["plotSource"], parms["plotTime"], \
plotFile, parms["refAnt"], err, \
Stokes=["RR","LL"], doband=-1, \
check=check, debug=debug, logfile=logFile )
if retCode != 0:
raise RuntimeError("Error in Plotting spectrum")
# Bandpass calibration
if parms["doBPCal"] and parms["BPCals"]:
retCode = EVLABPCal(uv, parms["BPCals"], err, noScrat=noScrat, solInt1=parms["bpsolint1"], \
solInt2=parms["bpsolint2"], solMode=parms["bpsolMode"], \
BChan1=parms["bpBChan1"], EChan1=parms["bpEChan1"], \
BChan2=parms["bpBChan2"], EChan2=parms["bpEChan2"], ChWid2=parms["bpChWid2"], \
doCenter1=parms["bpDoCenter1"], refAnt=parms["refAnt"], \
UVRange=parms["bpUVRange"], doCalib=2, gainUse=0, flagVer=2, doPlot=False, \
nThreads=nThreads, logfile=logFile, check=check, debug=debug)
if retCode != 0:
raise RuntimeError("Error in Bandpass calibration")
# Plot corrected data?
if parms["doSpecPlot"] and parms["plotSource"]:
plotFile = "./" + fileRoot + "BPSpec.ps"
retCode = EVLASpectrum(uv, parms["plotSource"], parms["plotTime"], plotFile, \
parms["refAnt"], err, Stokes=["RR","LL"], doband=1, \
check=check, debug=debug, logfile=logFile )
if retCode != 0:
raise RuntimeError("Error in Plotting spectrum")
# Constrain X/Y gain
if parms["doXYRelGain"] and (parms["XYRelGainCal"] != None):
mess = "Calibrate X/Y relative gain:"
printMess(mess, logFile)
plotFile = "./" + fileRoot + "XYGainCal.ps"
retCode = EVLACalAP (uv, [], parms["XYRelGainCal"], err,
doCalib=2, doBand=1, BPVer=1, flagVer=2, flagFail=False, \
BChan=parms["ampBChan"], EChan=parms["ampEChan"], \
solInt=parms["solInt"], solSmo=1440., ampScalar=parms["ampScalar"], \
doPlot=parms["doSNPlot"], plotFile=plotFile, refAnt=parms["refAnt"], \
nThreads=nThreads, noScrat=noScrat, logfile=logFile, check=check, debug=debug)
if retCode != 0:
raise RuntimeError("Error calibrating X/Y gain")
parms["avgPol"] = True
# Fix X/Y gain
# Amp & phase Calibrate
if parms["doAmpPhaseCal"]:
plotFile = "./" + fileRoot + "APCal.ps"
retCode = EVLACalAP (uv, [], parms["ACals"], err, PCals=parms["PCals"],
doCalib=2, doBand=1, BPVer=1, flagVer=2, avgPol=parms["avgPol"], \
BChan=parms["ampBChan"], EChan=parms["ampEChan"], \
solInt=parms["solInt"], solSmo=parms["solSmo"], ampScalar=parms["ampScalar"], \
doAmpEdit=parms["doAmpEdit"], ampSigma=parms["ampSigma"], \
ampEditFG=parms["ampEditFG"], \
doPlot=parms["doSNPlot"], plotFile=plotFile, refAnt=parms["refAnt"], \
nThreads=nThreads, noScrat=noScrat, logfile=logFile, check=check, debug=debug)
if retCode != 0:
raise RuntimeError("Error calibrating")
# More editing
if parms["doAutoFlag"]:
mess = "Post calibration editing:"
printMess(mess, logFile)
# if going to redo then only calibrators
if parms["doRecal"]:
# Only calibrators
clist = []
for DCal in DCals:
if DCal["Source"] not in clist:
clist.append(DCal["Source"])
for PCal in PCals:
if PCal["Source"] not in clist:
clist.append(DCal["Source"])
for ACal in ACals:
if ACal["Source"] not in clist:
clist.append(ACal["Source"])
else:
clist = []
retCode = EVLAAutoFlag (uv, clist, err, flagVer=2, \
doCalib=2, gainUse=0, doBand=1, BPVer=1, \
IClip=parms["IClip"], minAmp=parms["minAmp"], timeAvg=parms["timeAvg"], \
doFD=parms["doAFFD"], FDmaxAmp=parms["FDmaxAmp"], FDmaxV=parms["FDmaxV"], \
FDwidMW=parms["FDwidMW"], FDmaxRMS=parms["FDmaxRMS"], \
FDmaxRes=parms["FDmaxRes"], FDmaxResBL=parms["FDmaxResBL"], \
FDbaseSel=parms["FDbaseSel"], \
nThreads=nThreads, logfile=logFile, check=check, debug=debug)
if retCode != 0:
raise RuntimeError("Error in AutoFlag")
# Redo the calibration using new flagging?
if parms["doBPCal2"] == None:
parms["doBPCal2"] = parms["doBPCal"]
if parms["doDelayCal2"] == None:
parms["doDelayCal2"] = parms["doDelayCal2"]
if parms["doXYRelGain2"] == None:
parms["doXYRelGain2"] = parms["doXYRelGain"]
if parms["doAmpPhaseCal2"] == None:
parms["doAmpPhaseCal2"] = parms["doAmpPhaseCal"]
if parms["doAutoFlag2"] == None:
parms["doAutoFlagCal2"] = parms["doAutoFlag"]
if parms["doRecal"]:
mess = "Redo calibration:"
printMess(mess, logFile)
EVLAClearCal(uv,
err,
doGain=True,
doFlag=False,
doBP=True,
check=check,
logfile=logFile)
OErr.printErrMsg(err, "Error resetting calibration")
# Delay recalibration
if parms["doDelayCal2"] and parms["DCals"] and not check:
plotFile = "./" + fileRoot + "DelayCal2.ps"
retCode = EVLADelayCal(uv, parms["DCals"], err, \
BChan=parms["delayBChan"], EChan=parms["delayEChan"], \
doCalib=2, flagVer=2, doBand=-1, \
solInt=parms["delaySolInt"], smoTime=1.0/60.0, \
refAnts=[parms["refAnt"]], doTwo=parms["doTwo"], \
doZeroPhs=parms["delayZeroPhs"], \
doPlot=parms["doSNPlot"], plotFile=plotFile, \
nThreads=nThreads, noScrat=noScrat, \
logfile=logFile, check=check, debug=debug)
if retCode != 0:
raise RuntimeError("Error in delay calibration")
# Plot corrected data?
if parms["doSpecPlot"] and parms["plotSource"]:
plotFile = "./" + fileRoot + "DelaySpec2.ps"
retCode = EVLASpectrum(uv, parms["plotSource"], parms["plotTime"], plotFile, parms["refAnt"], err, \
Stokes=["RR","LL"], doband=-1, \
check=check, debug=debug, logfile=logFile )
if retCode != 0:
raise RuntimeError("Error in Plotting spectrum")
# Bandpass calibration
if parms["doBPCal2"] and parms["BPCals"]:
retCode = EVLABPCal(uv, parms["BPCals"], err, noScrat=noScrat, solInt1=parms["bpsolint1"], \
solInt2=parms["bpsolint2"], solMode=parms["bpsolMode"], \
BChan1=parms["bpBChan1"], EChan1=parms["bpEChan1"], \
BChan2=parms["bpBChan2"], EChan2=parms["bpEChan2"], ChWid2=parms["bpChWid2"], \
doCenter1=parms["bpDoCenter1"], refAnt=parms["refAnt"], \
UVRange=parms["bpUVRange"], doCalib=2, gainUse=0, flagVer=2, doPlot=False, \
nThreads=nThreads, logfile=logFile, check=check, debug=debug)
if retCode != 0:
raise RuntimeError("Error in Bandpass calibration")
# Plot corrected data?
if parms["doSpecPlot"] and parms["plotSource"]:
plotFile = "./" + fileRoot + "BPSpec2.ps"
retCode = EVLASpectrum(uv, parms["plotSource"], parms["plotTime"], plotFile, parms["refAnt"], err, \
Stokes=["RR","LL"], doband=1, \
check=check, debug=debug, logfile=logFile )
if retCode != 0:
raise RuntimeError("Error in Plotting spectrum")
# Constrain X/Y gain
parms["avgPol"] = False
# Fix X/Y gain?
if parms["doXYRelGain2"] and (parms["XYRelGainCal"] != None):
mess = "Calibrate X/Y relative gain:"
printMess(mess, logFile)
plotFile = "./" + fileRoot + "XYGainCal2.ps"
retCode = EVLACalAP (uv, [], parms["XYRelGainCal"], err,
doCalib=2, doBand=1, BPVer=1, flagVer=2, timeRange=parms["XYRelGainCalTime"], \
BChan=parms["ampBChan"], EChan=parms["ampEChan"], flagFail=False, \
solInt=parms["solInt"], solSmo=1440., ampScalar=parms["ampScalar"], \
doPlot=parms["doSNPlot"], plotFile=plotFile, refAnt=parms["refAnt"], \
nThreads=nThreads, noScrat=noScrat, logfile=logFile, check=check, debug=debug)
if retCode != 0:
raise RuntimeError("Error calibrating X/Y gain")
parms["avgPol"] = True
# Fix X/Y gain
# Amp & phase Recalibrate
if parms["doAmpPhaseCal2"]:
plotFile = "./" + fileRoot + "APCal2.ps"
retCode = EVLACalAP (uv, [], parms["ACals"], err, PCals=parms["PCals"], \
doCalib=2, doBand=1, BPVer=1, flagVer=2,avgPol=parms["avgPol"], \
BChan=parms["ampBChan"], EChan=parms["ampEChan"], \
solInt=parms["solInt"], solSmo=parms["solSmo"], ampScalar=parms["ampScalar"], \
doAmpEdit=parms["doAmpEdit"], ampSigma=parms["ampSigma"], \
ampEditFG=parms["ampEditFG"], \
doPlot=parms["doSNPlot"], plotFile=plotFile, refAnt=parms["refAnt"], \
noScrat=noScrat, nThreads=nThreads, logfile=logFile, check=check, debug=debug)
if retCode != 0:
raise RuntimeError("Error calibrating")
# More editing
if parms["doAutoFlag2"]:
mess = "Post recalibration editing:"
printMess(mess, logFile)
retCode = EVLAAutoFlag (uv, [], err, flagVer=2, \
doCalib=2, gainUse=0, doBand=1, BPVer=1, \
IClip=parms["IClip"], minAmp=parms["minAmp"], timeAvg=parms["timeAvg"], \
doFD=parms["doAFFD"], FDmaxAmp=parms["FDmaxAmp"], FDmaxV=parms["FDmaxV"], \
FDwidMW=parms["FDwidMW"], FDmaxRMS=parms["FDmaxRMS"], \
FDmaxRes=parms["FDmaxRes"], FDmaxResBL= parms["FDmaxResBL"], \
FDbaseSel=parms["FDbaseSel"], \
nThreads=nThreads, logfile=logFile, check=check, debug=debug)
if retCode != 0:
raise RuntimeError("Error in AutoFlag")
# end recal
# Calibrate and average data
if parms["doCalAvg"]:
retCode = EVLACalAvg (uv, avgClass, parms["seq"], parms["CalAvgTime"], err, \
flagVer=2, doCalib=2, gainUse=0, doBand=1, BPVer=1, doPol=False, \
avgFreq=parms["avgFreq"], chAvg=parms["chAvg"], \
BChan=parms["CABChan"], EChan=parms["CAEChan"], \
BIF=parms["CABIF"], EIF=parms["CAEIF"], Compress=parms["Compress"], \
nThreads=nThreads, logfile=logFile, check=check, debug=debug)
if retCode != 0:
raise RuntimeError("Error in CalAvg")
# Get calibrated/averaged data
if not check:
uv = UV.newPAUV("AIPS UV DATA", EVLAAIPSName(project, session), avgClass[0:6], \
disk, parms["seq"], True, err)
if err.isErr:
OErr.printErrMsg(err, "Error creating cal/avg AIPS data")
# XClip
if parms["XClip"] and parms["XClip"] > 0.0:
mess = "Cross Pol clipping:"
printMess(mess, logFile)
retCode = EVLAAutoFlag (uv, [], err, flagVer=-1, flagTab=1, \
doCalib=2, gainUse=0, doBand=-1, maxBad=1.0, \
XClip=parms["XClip"], timeAvg=1./60., \
nThreads=nThreads, logfile=logFile, check=check, debug=debug)
if retCode != 0:
raise RuntimeError("Error in AutoFlag")
# R-L delay calibration cal if needed,
if parms["doRLDelay"] and parms["RLDCal"][0][0] != None:
if parms["rlrefAnt"] <= 0:
parms["rlrefAnt"] = parms["refAnt"]
# parms["rlDoBand"] if before average, BPVer=parms["rlBPVer"],
retCode = EVLARLDelay(uv, err,\
RLDCal=parms["RLDCal"], BChan=parms["rlBChan"], \
EChan=parms["rlEChan"], UVRange=parms["rlUVRange"], \
soucode=parms["rlCalCode"], doCalib=parms["rlDoCal"], gainUse=parms["rlgainUse"], \
timerange=parms["rltimerange"], \
# NOT HERE doBand=parms["rlDoBand"], BPVer=parms["rlBPVer"], \
flagVer=parms["rlflagVer"], \
refAnt=parms["rlrefAnt"], doPol=False, \
nThreads=nThreads, noScrat=noScrat, logfile=logFile, \
check=check, debug=debug)
if retCode != 0:
raise RuntimeError("Error in R-L delay calibration")
# Polarization calibration
if parms["doPolCal"]:
if parms["PCRefAnt"] <= 0:
parms["PCRefAnt"] = parms["refAnt"]
retCode = EVLAPolCal(uv, parms["PCInsCals"], err, \
doCalib=2, gainUse=0, doBand=-1, flagVer=0, \
fixPoln=parms["PCFixPoln"], pmodel=parms["PCpmodel"], avgIF=parms["PCAvgIF"], \
solInt=parms["PCSolInt"], refAnt=parms["PCRefAnt"], solType=parms["PCSolType"], \
ChInc=parms["PCChInc"], ChWid=parms["PCChWid"], \
nThreads=nThreads, check=check, debug=debug, noScrat=noScrat, logfile=logFile)
if retCode != 0 and (not check):
raise RuntimeError("Error in polarization calibration: " +
str(retCode))
# end poln cal.
# R-L phase calibration cal., creates new BP table
if parms["doRLCal"] and parms["RLDCal"][0][0] != None:
plotFile = "./" + fileRoot + "RLSpec2.ps"
if parms["rlrefAnt"] <= 0:
parms["rlrefAnt"] = parms["refAnt"]
retCode = EVLARLCal(uv, err,\
RLDCal=parms["RLDCal"], BChan=parms["rlBChan"],
EChan=parms["rlEChan"], UVRange=parms["rlUVRange"], \
ChWid2=parms["rlChWid"], solInt1=parms["rlsolint1"], solInt2=parms["rlsolint2"], \
RLPCal=parms["RLPCal"], RLPhase=parms["RLPhase"], \
RM=parms["RLRM"], CleanRad=parms["rlCleanRad"], \
calcode=parms["rlCalCode"], doCalib=parms["rlDoCal"], gainUse=parms["rlgainUse"], \
timerange=parms["rltimerange"], FOV=parms["rlFOV"], \
doBand=-1, BPVer=1, flagVer=parms["rlflagVer"], \
refAnt=parms["rlrefAnt"], doPol=parms["doPol"], PDVer=parms["PDVer"], \
doPlot=parms["doSpecPlot"], plotFile=plotFile, \
nThreads=nThreads, noScrat=noScrat, logfile=logFile, \
check=check, debug=debug)
if retCode != 0:
raise RuntimeError("Error in RL phase spectrum calibration")
# VClip
if parms["VClip"] and parms["VClip"] > 0.0:
mess = "VPol clipping:"
printMess(mess, logFile)
retCode = EVLAAutoFlag (uv, [], err, flagVer=-1, flagTab=1, \
doCalib=2, gainUse=0, doBand=-1, \
VClip=parms["VClip"], timeAvg=parms["timeAvg"], \
nThreads=nThreads, logfile=logFile, check=check, debug=debug)
if retCode != 0:
raise RuntimeError("Error in AutoFlag VClip")
# Plot corrected data?
if parms["doSpecPlot"] and parms["plotSource"]:
plotFile = "./" + fileRoot + "Spec.ps"
retCode = EVLASpectrum(uv, parms["plotSource"], parms["plotTime"], \
plotFile, parms["refAnt"], err, \
Stokes=["RR","LL"], doband=-1, \
check=check, debug=debug, logfile=logFile )
if retCode != 0:
raise RuntimeError("Error in Plotting spectrum")
# Image targets
if parms["doImage"]:
# If targets not specified, image all
if len(parms["targets"]) <= 0:
slist = EVLAAllSource(uv,
err,
logfile=logFile,
check=check,
debug=debug)
else:
slist = parms["targets"]
EVLAImageTargets (uv, err, Sources=slist, seq=parms["seq"], sclass=outIClass, \
doCalib=2, doBand=1, flagVer=1, doPol=parms["doPol"], PDVer=parms["PDVer"], \
Stokes=parms["Stokes"], FOV=parms["FOV"], Robust=parms["Robust"], Niter=parms["Niter"], \
CleanRad=parms["CleanRad"], minFlux=parms["minFlux"], \
maxPSCLoop=parms["maxPSCLoop"], minFluxPSC=parms["minFluxPSC"], \
solPInt=parms["solPInt"], solPMode=parms["solPMode"], solPType=parms["solPType"], \
maxASCLoop=parms["maxASCLoop"], minFluxASC=parms["minFluxASC"], \
solAInt=parms["solAInt"], solAMode=parms["solAMode"], solAType=parms["solAType"], \
avgPol=parms["avgPol"], avgIF=parms["avgIF"], minSNR = 4.0, refAnt=parms["refAnt"], \
do3D=parms["do3D"], BLFact=parms["BLFact"], BLchAvg=parms["BLchAvg"], \
doMB=parms["doMB"], norder=parms["MBnorder"], maxFBW=parms["MBmaxFBW"], \
PBCor=parms["PBCor"],antSize=parms["antSize"], Beam=parms["Beam"], \
nTaper=parms["nTaper"], Tapers=parms["Tapers"], \
nThreads=nThreads, noScrat=noScrat, logfile=logFile, check=check, debug=debug)
# End image
# Get report on sources
if parms["doReport"]:
# If targets not specified, do all
if len(parms["targets"]) <= 0:
slist = EVLAAllSource(uv,
err,
logfile=logFile,
check=check,
debug=debug)
else:
slist = parms["targets"]
Report = EVLAReportTargets(uv, err, Sources=slist, seq=parms["seq"], sclass=outIClass, \
Stokes=parms["Stokes"], logfile=logFile, check=check, debug=debug)
# Save to pickle jar
ReportPicklefile = "./" + fileRoot + "Report.pickle" # Where results saved
SaveObject(Report, ReportPicklefile, True)
# Write results, cleanup
# Save cal/average UV data?
if parms["doSaveUV"] and (not check):
Aname = EVLAAIPSName(project, session)
cno = AIPSDir.PTestCNO(disk, user, Aname, avgClass[0:6], "UV",
parms["seq"], err)
if cno > 0:
uvt = UV.newPAUV("AIPS CAL UV DATA", Aname, avgClass, disk,
parms["seq"], True, err)
filename = parms["project"] + parms["session"] + parms[
"band"] + "Cal.uvtab"
fuv = EVLAUVFITS(uvt,
filename,
0,
err,
compress=parms["Compress"],
logfile=logFile)
EVLAAddOutFile(filename, 'project', "Calibrated Averaged UV data")
# Save list of output files
EVLASaveOutFiles()
del uvt
# Save raw UV data tables?
if parms["doSaveTab"] and (not check):
Aname = EVLAAIPSName(project, session)
cno = AIPSDir.PTestCNO(disk, user, Aname, dataClass[0:6], "UV",
parms["seq"], err)
if cno > 0:
uvt = UV.newPAUV("AIPS RAW UV DATA", Aname, dataClass[0:6], disk,
parms["seq"], True, err)
filename = parms["project"] + parms["session"] + parms[
"band"] + "CalTab.uvtab"
fuv = EVLAUVFITSTab(uvt, filename, 0, err, logfile=logFile)
EVLAAddOutFile(filename, 'project', "Calibrated AIPS tables")
del uvt
# Write History
filename = project + '_' + session + '_' + band + ".History.text"
OTObit.PrintHistory(uv, file=filename)
EVLAAddOutFile(filename, 'project',
"Processing history of calibrated data")
# Save list of output files
EVLASaveOutFiles()
# Imaging results
# If targets not specified, save all
if len(parms["targets"]) <= 0:
slist = EVLAAllSource(uv,
err,
logfile=logFile,
check=check,
debug=debug)
else:
slist = parms["targets"]
for target in slist:
if parms["doSaveImg"] and (not check):
for s in parms["Stokes"]:
oclass = s + outIClass[1:]
outname = target
# Test if image exists
cno = AIPSDir.PTestCNO(disk, user, outname, oclass, "MA",
parms["seq"], err)
if cno <= 0:
continue
x = Image.newPAImage("out", outname, oclass, disk,
parms["seq"], True, err)
outfile = "./" + fileRoot + target + "." + oclass + ".fits"
xf = EVLAImFITS(x, outfile, 0, err, logfile=logFile)
EVLAAddOutFile(outfile, target, 'Image of ' + target)
# Statistics
zz = imstat(x, err, logfile=logFile)
# end writing loop
# Save list of output files
EVLASaveOutFiles()
OErr.printErrMsg(err, "Writing output")
# Contour plots
if parms["doKntrPlots"]:
mess = "INFO --> Contour plots (doKntrPlots)"
printMess(mess, logFile)
EVLAKntrPlots(err,
imName=parms["targets"],
project=project,
session=session,
band=band,
disk=disk,
debug=debug)
# Save list of output files
EVLASaveOutFiles()
elif debug:
mess = "Not creating contour plots ( doKntrPlots = " + str(
parms["doKntrPlots"]) + " )"
printMess(mess, logFile)
# Source uv plane diagnostic plots
if parms["doDiagPlots"]:
mess = "INFO --> Diagnostic plots (doDiagPlots)"
printMess(mess, logFile)
# Get the highest number avgClass catalog file
Aname = EVLAAIPSName(project, session)
uvc = None
if not check:
uvname = project + "_" + session + "_" + band + "_Cal"
uvc = UV.newPAUV(uvname, Aname, avgClass, disk, parms["seq"], True,
err)
EVLADiagPlots( uvc, err, cleanUp=parms["doCleanup"], \
project=project, session=session, band=band, \
logfile=logFile, check=check, debug=debug )
# Save list of output files
EVLASaveOutFiles()
elif debug:
mess = "Not creating diagnostic plots ( doDiagPlots = " + str(
parms["doDiagPlots"]) + " )"
printMess(mess, logFile)
# Save metadata
srcMetadata = None
projMetadata = None
if parms["doMetadata"]:
mess = "INFO --> Save metadata (doMetadata)"
printMess(mess, logFile)
uvc = None
if not uvc:
# Get calibrated/averaged data
Aname = EVLAAIPSName(project, session)
uvname = project + "_" + session + "_" + band + "_Cal"
uvc = UV.newPAUV(uvname, Aname, avgClass, disk, parms["seq"], True,
err)
if err.isErr:
OErr.printErrMsg(err, "Error creating cal/avg AIPS data")
# Get source metadata; save to pickle file
srcMetadata = EVLASrcMetadata( uvc, err, Sources=parms["targets"], seq=parms["seq"], \
sclass=outIClass, Stokes=parms["Stokes"],\
logfile=logFile, check=check, debug=debug )
picklefile = "./" + fileRoot + ".SrcReport.pickle"
SaveObject(srcMetadata, picklefile, True)
EVLAAddOutFile(picklefile, 'project', 'All source metadata')
# Get project metadata; save to pickle file
projMetadata = EVLAProjMetadata( uvc, AIPS_VERSION, err, \
PCals=parms["PCals"], ACals=parms["ACals"], \
BPCals=parms["BPCals"], DCals=parms["DCals"], \
project = project, session = session, band = band, \
dataInUVF = parms["archRoot"], archFileID = 66666 )
picklefile = "./" + fileRoot + ".ProjReport.pickle"
SaveObject(projMetadata, picklefile, True)
EVLAAddOutFile(picklefile, 'project', 'Project metadata')
else:
# Fetch from pickle jar
picklefile = "./" + fileRoot + ".SrcReport.pickle"
srcMetadata = FetchObject(picklefile)
picklefile = "./" + fileRoot + ".ProjReport.pickle"
projMetadata = FetchObject(picklefile)
# Write report
if parms["doHTML"]:
mess = "INFO --> Write HTML report (doHTML)"
printMess(mess, logFile)
EVLAHTMLReport( projMetadata, srcMetadata, \
outfile="./"+fileRoot+".report.html", \
logFile=logFile )
# Write VOTable
if parms["doVOTable"]:
mess = "INFO --> Write VOTable (doVOTable)"
printMess(mess, logFile)
EVLAAddOutFile('VOTable.xml', 'project', 'VOTable report')
EVLAWriteVOTable(projMetadata, srcMetadata, filename='VOTable.xml')
# Save list of output files
EVLASaveOutFiles()
# Cleanup - delete AIPS files
if parms["doCleanup"] and (not check):
mess = "INFO --> Clean up (doCleanup)"
printMess(mess, logFile)
# Delete target images
# How many Stokes images
nstok = len(parms["Stokes"])
for istok in range(0, nstok):
oclass = parms["Stokes"][istok:istok + 1] + outIClass[1:]
AllDest(err, disk=disk, Aseq=parms["seq"], Aclass=oclass)
# Delete initial UV data
Aname = EVLAAIPSName(project, session)
# Test if data exists
cno = AIPSDir.PTestCNO(disk, user, Aname, dataClass[0:6], "UV",
parms["seq"], err)
if cno > 0:
uvt = UV.newPAUV("AIPS RAW UV DATA", Aname, dataClass[0:6], disk,
parms["seq"], True, err)
uvt.Zap(err)
del uvt
if err.isErr:
OErr.printErrMsg(err, "Error deleting raw AIPS data")
# Zap calibrated/averaged data
# Test if data exists
cno = AIPSDir.PTestCNO(disk, user, Aname, avgClass[0:6], "UV",
parms["seq"], err)
if cno > 0:
uvt = UV.newPAUV("AIPS CAL UV DATA", Aname, avgClass[0:6], disk,
parms["seq"], True, err)
uvt.Zap(err)
del uvt
if err.isErr:
OErr.printErrMsg(err, "Error deleting cal/avg AIPS data")
# Zap UnHanned data if present
loadClass = parms["band"] + "Raw"
# Test if image exists
cno = AIPSDir.PTestCNO(disk, user, Aname, loadClass[0:6], "UV",
parms["seq"], err)
if cno > 0:
uvt = UV.newPAUV("AIPS CAL UV DATA", Aname, loadClass[0:6], disk,
parms["seq"], True, err)
uvt.Zap(err)
del uvt
if err.isErr:
OErr.printErrMsg(err, "Error deleting cal/avg AIPS data")
OErr.printErrMsg(err, "Writing output/cleanup")
# Shutdown
mess = "Finished project "+parms["project"]+" session "+parms["session"]+ \
" "+parms["band"]+" Band"+" AIPS user no. "+str(AIPS.userno)
printMess(mess, logFile)
OErr.printErr(err)
OSystem.Shutdown(ObitSys)
doBP=doBP,
check=check,
logfile=logFile)
OErr.printErrMsg(err, "Error resetting calibration")
# Copy FG 1 to FG 2
if doCopyFG:
retCode = VLBACopyFG(uv, err, logfile=logFile, check=check, debug=debug)
if retCode != 0:
raise RuntimeError, "Error Copying FG table"
# Special editing
if doEditList and not check:
for edt in editList:
UV.PFlag(uv,err,timeRange=[dhms2day(edt["timer"][0]),dhms2day(edt["timer"][1])], \
flagVer=editFG, Ants=edt["Ant"], Chans=edt["Chans"], IFs=edt["IFs"], \
Stokes=edt["Stokes"], Reason=edt["Reason"])
OErr.printErrMsg(err, "Error Flagging")
# Quack to remove data from start and end of each scan
if doQuack:
retCode = VLBAQuack (uv, err, begDrop=quackBegDrop, endDrop=quackEndDrop, Reason=quackReason, \
logfile=logFile, check=check, debug=debug)
if retCode != 0:
raise RuntimeError, "Error Quacking data"
# Add velocity/rest freq info
if doFreqInfo:
retCode = VLBAFreqInfo(uv,
restfreq,
srcVel,
def pipeline(aipsSetup, parmFile):
"""
VLBA Continuum pipeline.
* *aipsSetup* = AIPS setup file
* *parmFile* = pipeline input parameters file
"""
############################# Initialize OBIT ##########################################
noScrat = []
exec(open(aipsSetup).read())
VLBAAddOutFile(aipsSetup, 'project', "Obit's AIPS setup file")
############################# Default parameters ##########################################
# Define data
project = "Unspecified" # Project name (12 char or less, used as AIPS Name)
session = "?" # Project session code
band = "?" # Observing band
logFile = project + "_" + session + "_" + band + ".log" # Processing log file
seq = 1 # AIPS sequence number
gain = 0.10 # CLEAN loop gain
doLoadIDI = True # Load data from IDI FITS?
doLoadUVF = False # Load the "AIPS Friendly" (uvfits) FITS version
dataInUVF = None # Input uvfits data file name
dataInIDI = None # Input FITS-IDI file or list
dataClass = "Raw" # AIPS class of raw uv data
Compress = False # Use compressed UV data?
calInt = 0.15 # Calibration table interval in min.
wtThresh = 0.8 # Data weight threshold
check = False # Only check script, don't execute tasks
debug = False # run tasks debug
prtLv = 2 # Print level
# Initialize parameters
parms = VLBAInitContParms()
############################# Set Project Processing parameters ##################
exec(open(parmFile).read())
VLBAAddOutFile(parmFile, 'project', 'Pipeline input parameters')
################################## Process #####################################
# Init cal pickle jars
goodCalPicklefile = project + "_" + session + "_" + band + "_GoodCal.pickle" # Where results saved
# Default "best" calibration
goodCal = {
"Source": " ",
"souID": 0,
"timeRange": (0.0, 100.0),
"Fract": 0.0,
"SNR": 0.0,
"bestRef": 0
}
# Save initial default if it doesn't already exist
SaveObject(goodCal, goodCalPicklefile, False)
VLBAAddOutFile(goodCalPicklefile, 'project', 'Best calibrator information')
OKCalPicklefile = project + "_" + session + "_" + band + "_OKCal.pickle" # Where results saved
SaveObject(parms["contCals"], OKCalPicklefile, False)
VLBAAddOutFile(OKCalPicklefile, 'project', 'List of calibrators')
targetsPicklefile = project + "_" + session + "_" + band + "_targets.pickle" # Where results saved
if (not parms["targets"]) and os.path.exists(targetsPicklefile):
parms["targets"] = FetchObject(targetsPicklefile)
else:
SaveObject(parms["targets"], targetsPicklefile, False)
VLBAAddOutFile(targetsPicklefile, 'project', 'List of targets')
# Load the outputs pickle jar
VLBAFetchOutFiles()
# Logging directly to logFile
OErr.PInit(err, prtLv, logFile)
retCode = 0
VLBAAddOutFile(logFile, 'project', 'Pipeline log file')
mess = "Start project " + project + " session " + session + " " + band + " Band" + " AIPS user no. " + str(
AIPS.userno)
printMess(mess, logFile)
logger.info(mess)
if debug:
pydoc.ttypager = pydoc.plainpager # don't page task input displays
mess = "Using Debug mode "
printMess(mess, logFile)
if check:
mess = "Only checking script"
printMess(mess, logFile)
# Load Data from FITS
uv = None # Raw data
uvc = None # Cal/averaged data
if doLoadIDI:
logger.info("--> Load IDI data file (doLoadIDI)")
if type(dataInIDI) == list:
logger.info(
"--> Loading a list of IDI files (dataInIDI is a list)\n" +
" This data appears to be from the old correlator.\n" +
" IDI corrections will be applied when loading is\n" +
" complete.")
# Read list
for dataIn in dataInIDI:
uv = VLBAIDILoad(dataIn, project, session, band, dataClass, disk, seq, err, logfile=logFile, \
wtThresh=wtThresh, calInt=calInt, Compress=Compress, \
check=check, debug=debug)
if not UV.PIsA(uv):
raise RuntimeError("Cannot load " + dataIn)
# Fix IDI files:
uv = IDIFix.IDIFix(uv, err)
seq = uv.Aseq
else:
logger.info(
"--> Loading a single IDI file (dataInIDI is not a list)\n" +
" This data appears to be from the DiFX correlator.\n" +
" No IDI corrections will be applied.")
# Single IDI file
uv = VLBAIDILoad(dataInIDI, project, session, band, dataClass, disk, seq, err, logfile=logFile, \
wtThresh=wtThresh, calInt=calInt, Compress=Compress, \
check=check, debug=debug)
if not UV.PIsA(uv):
raise RuntimeError("Cannot load " + dataInIDI)
if doLoadUVF:
logger.info("--> Load UVFITS data file (doLoadUVF)")
uv = VLBAIDILoad(dataInUVF, project, session, band, dataClass, disk, seq, err, logfile=logFile, \
wtThresh=wtThresh, calInt=calInt, Compress=Compress, \
check=check, debug=debug)
# Adding check condition to avoid error when checking
if not UV.PIsA(uv) and not check:
raise RuntimeError("Cannot load " + dataInUVF)
# Otherwise set uv
if uv == None and not check:
Aname = VLBAAIPSName(project, session)
uvname = project + "_" + session + "_" + band
uv = UV.newPAUV(uvname, Aname, dataClass, disk, seq, True, err)
if err.isErr:
OErr.printErrMsg(err, "Error creating AIPS data")
# frequency dependent default parameters
VLBAInitContFQParms(uv, parms, err, \
logfile=logFile, check=check, debug=debug)
# Setup target source list
if os.path.exists(targetsPicklefile):
parms["targets"] = \
setupSourceList( parms["targets"], uv, err, logFile, check, debug )
logger.debug("parms['targets'] = " + str(parms["targets"]))
# Clear any old calibration/editing
if parms["doClearTab"]:
logger.info("--> Clear old calibration/editing (doClearTab)")
VLBAClearCal(uv, err, doGain=parms["doGain"], doFlag=parms["doFlag"], doBP=parms["doBP"], \
check=check, logfile=logFile)
OErr.printErrMsg(err, "Error resetting calibration")
# Copy FG 1 to FG 2
if parms["doCopyFG"]:
logger.info("--> Copy flag (FG) table 1 to flag table 2 (doCopyFG)")
retCode = VLBACopyFG(uv,
err,
logfile=logFile,
check=check,
debug=debug)
if retCode != 0:
raise RuntimeError("Error Copying FG table")
# Special editing
if parms["doEditList"] and not check:
logger.info("--> Special editing (doEditList)")
for edt in parms["editList"]:
UV.PFlag(uv,err,timeRange=[dhms2day(edt["timer"][0]),dhms2day(edt["timer"][1])], \
flagVer=editFG, Ants=edt["Ant"], Chans=edt["Chans"], IFs=edt["IFs"], \
Stokes=edt["Stokes"], Reason=edt["Reason"])
OErr.printErrMsg(err, "Error Flagging")
# Quack to remove data from start and end of each scan
if parms["doQuack"]:
logger.info("--> Remove data from start and end of scans (doQuack)")
retCode = VLBAQuack (uv, err, \
begDrop=parms["quackBegDrop"], endDrop=parms["quackEndDrop"], \
Reason=parms["quackReason"], \
logfile=logFile, check=check, debug=debug)
if retCode != 0:
raise RuntimeError("Error Quacking data")
# Median window time editing, for RFI impulsive in time or dropouts
if parms["doMedn"]:
logger.info("--> Median window time editing (doMedn)")
retCode = VLBAMedianFlag (uv, " ", err, noScrat=noScrat, nThreads=nThreads, \
avgTime=parms["mednAvgTime"], avgFreq=parms["mednAvgFreq"], \
chAvg=parms["mednChAvg"], timeWind=parms["mednTimeWind"], \
flagSig=parms["mednSigma"], flagVer=2, \
logfile=logFile, check=check, debug=False)
if retCode != 0:
raise RuntimeError("Error in MednFlag")
# Quantization correction?
if parms["doQuantCor"]:
logger.info("--> Quantization correction (doQuantCor)")
plotFile = project + "_" + session + "_" + band + ".Quant.ps"
retCode = VLBAQuantCor(uv, parms["QuantSmo"], parms["QuantFlag"], err, \
doSNPlot=parms["doSNPlot"], plotFile=plotFile, \
logfile=logFile, check=check, debug=debug)
if retCode != 0:
raise RuntimeError("Error in quantization correcting/flagging")
# Parallactic angle correction?
if parms["doPACor"]:
logger.info("--> Parallactic angle correction (doPACor)")
retCode = VLBAPACor(uv, err, noScrat=noScrat, \
logfile=logFile, check=check, debug=debug)
if retCode != 0:
raise RuntimeError("Error in quantization correcting/flagging")
# Opacity/Tsys/gain correction
if parms["doOpacCor"]:
logger.info("--> Opacity/Tsys/Gain correction (doOpacCor)")
plotFile = project + "_" + session + "_" + band + ".Opacity.ps"
retCode = VLBAOpacCor(uv, parms["OpacSmoo"], err, \
doSNPlot=parms["doSNPlot"], plotFile=plotFile, \
logfile=logFile, check=check, debug=debug)
if retCode != 0:
raise RuntimeError("Error in opacity/gain.Tsys correction")
VLBASaveOutFiles() # Save plot file in Outfiles
# Need to determine a list of calibrators?
if (parms["contCals"] == None) or (len(parms["contCals"]) <= 0):
logger.info("--> Get list of calibrators (contCals = None or [])")
if parms["doFindOK"]:
slist = VLBAAllSource(uv,
err,
logfile=logFile,
check=check,
debug=debug)
parms["contCals"] = VLBAOKCal(uv, parms["minOKFract"], err, \
solInt=parms["findSolInt"], \
calSou=slist, minSNR=parms["minOKSNR"], \
doCalib=-1, flagVer=2, refAnts=parms["refAnts"], \
noScrat=noScrat, nThreads=nThreads, \
logfile=logFile, check=check, debug=debug)
if not parms["contCals"] and not check:
raise RuntimeError("Error in finding acceptable calibrators")
logger.info("Calibrators = " + str(parms["contCals"]))
else:
# Snatch from pickle jar
parms["contCals"] = FetchObject(OKCalPicklefile)
# Save contCals to a pickle jar
SaveObject(parms["contCals"], OKCalPicklefile, True)
# Find best calibration source
if parms["doFindCal"]:
logger.info("--> Find best calibration source (doFindCal)")
goodCal = VLBAGoodCal(uv, err, \
solInt=parms["findSolInt"], timeInt=parms["findTimeInt"], \
calSou=parms["contCals"], \
#CalModel=parms["contCalModel"], \
doCalib=-1, flagVer=2, refAnts=parms["refAnts"], \
noScrat=noScrat, nThreads=nThreads, \
logfile=logFile, check=check, debug=debug)
if not goodCal and not check:
raise RuntimeError("Error in finding best calibration data")
# Save it to a pickle jar
SaveObject(goodCal, goodCalPicklefile, True)
else:
# Fetch from pickle
goodCal = FetchObject(goodCalPicklefile)
# Apply Phase cals from PC table?
if parms["doPCcor"] and not check:
logger.info("--> Apply phase cals (doPCcor)")
plotFile = project + "_" + session + "_" + band + ".PC.ps"
retCode = VLBAPCcor(uv, err, calSou=goodCal["Source"], \
timeRange=goodCal["timeRange"], \
doCalib=-1, flagVer=2, solInt=parms["manPCsolInt"], \
PCin=1, SNout=0, refAnt=goodCal["bestRef"], \
doPCPlot=parms["doPCPlot"], plotFile=plotFile, \
noScrat=noScrat, logfile=logFile, check=check, debug=debug)
if retCode != 0:
raise RuntimeError("Error in PC calibration")
VLBASaveOutFiles() # Save plot file in Outfiles
# manual phase cal
if parms["doManPCal"] and not check:
logger.info("--> Manual phase cal (doManPCal)")
plotFile = project + session + band + ".ManPCal.ps"
retCode = VLBAManPCal(uv, err, calSou=goodCal["Source"], \
#CalModel=parms["contCalModel"], \
timeRange=goodCal["timeRange"], \
solInt=parms["manPCsolInt"], smoTime=parms["manPCSmoo"], \
refAnts=[goodCal["bestRef"]], doCalib=2, flagVer=2, \
doManPCalPlot=parms["doManPCalPlot"], \
plotFile=plotFile, noScrat=noScrat, \
nThreads=nThreads, logfile=logFile, check=check, debug=debug)
if retCode != 0:
raise RuntimeError("Error in manual phase calibration")
# image cals
if parms["doImgCal"] and not check:
logger.info("--> Image calibrators (doImgCal)")
VLBAImageCals(uv,
err,
Sources=parms["contCals"],
seq=seq,
sclass=parms["outCclass"],
doCalib=2,
flagVer=2,
doBand=0,
FOV=parms["FOV"],
Robust=parms["Robust"],
maxPSCLoop=parms["maxPSCLoop"],
minFluxPSC=parms["minFluxPSC"],
solPInt=parms["solPInt"],
solMode=parms["solMode"],
maxASCLoop=parms["maxASCLoop"],
minFluxASC=parms["minFluxASC"],
solAInt=parms["solAInt"],
avgPol=parms["avgPol"],
avgIF=parms["avgIF"],
minSNR=parms["minSNR"],
refAnt=goodCal["bestRef"],
nThreads=nThreads,
noScrat=noScrat,
logfile=logFile,
check=check,
debug=debug)
# Rewrite OKCal pickle file because calibrators may have been updated
SaveObject(parms["contCals"], OKCalPicklefile, True)
if len(parms["contCals"]) <= 0:
logger.error(
"No calibration sources have been detected! Stopping pipeline."
)
raise RuntimeError("No calibration sources have been detected!")
# Check if calibrator models now available
parms["contCalModel"] = VLBAImageModel(parms["contCals"],
parms["outCclass"], disk, seq, err)
# delay calibration
if parms["doDelayCal"] and not check:
logger.info("--> Delay calibration (doDelayCal)")
plotFile = project + "_" + session + "_" + band + ".DelayCal.ps"
retCode = VLBADelayCal(uv, err, calSou=parms["contCals"], CalModel=parms["contCalModel"], \
doCalib=2, flagVer=2, doBand=0, \
solInt=parms["manPCsolInt"], smoTime=parms["delaySmoo"], \
refAnts=[goodCal["bestRef"]], \
doSNPlot=parms["doSNPlot"], plotFile=plotFile, \
nThreads=nThreads, noScrat=noScrat, logfile=logFile, check=check, debug=debug)
if retCode != 0:
raise RuntimeError("Error in delay calibration")
VLBASaveOutFiles() # Save plot file in Outfiles
# Bandpass calibration if needed
if parms["doBPCal"] and not check:
logger.info("--> Bandpass calibration (doBPCal)")
retCode = VLBABPass(uv, goodCal["Source"], err, CalModel=parms["contCalModel"], \
timeRange=goodCal["timeRange"], doCalib=2, flagVer=2, \
noScrat=noScrat, solInt1=parms["bpsolint1"], \
solInt2=parms["bpsolint2"], solMode=parms["bpsolMode"], \
BChan1=parms["bpBChan1"], EChan1=parms["bpEChan1"], BChan2=parms["bpBChan2"], \
EChan2=parms["bpEChan2"], ChWid2=parms["bpChWid2"], \
doCenter1=parms["bpDoCenter1"], refAnt=goodCal["bestRef"], specIndex=parms["specIndex"], \
doAuto = parms["bpdoAuto"], \
nThreads=nThreads, logfile=logFile, check=check, debug=debug)
if retCode != 0:
raise RuntimeError("Error in Bandpass calibration")
# Plot amplitude and phase vs. frequency
if parms["doSpecPlot"]:
logger.info("--> Spectral plotting (doSpecPlot)")
plotFile = project + '_' + session + '_' + band + ".spec.ps"
VLBASpecPlot(uv,
goodCal,
err,
doband=1,
check=check,
plotFile=plotFile,
logfile=logFile,
debug=debug)
VLBASaveOutFiles() # Save plot file in Outfiles
# Phase calibration using calibrator models
if parms["doPhaseCal"]:
logger.info(
"--> Phase calibration using calibrator models (doPhaseCal)")
plotFile = project + "_" + session + "_" + band + ".PhaseCal0.ps"
retCode = VLBAPhaseCal(uv, err, calSou=parms["contCals"], CalModel=parms["contCalModel"], \
doCalib=-1, flagVer=0, doBand=-1, \
refAnt=goodCal["bestRef"], solInt=parms["solPInt"], \
doSNPlot=parms["doSNPlot"], plotFile=plotFile, \
nThreads=nThreads, noScrat=noScrat, logfile=logFile, check=check, debug=debug)
if retCode != 0:
raise RuntimeError("Error in phase calibration")
VLBASaveOutFiles() # Save plot file in Outfiles
# Rewrite contCals pickle file because contCals may have been updated
SaveObject(parms["contCals"], OKCalPicklefile, True)
if len(parms["contCals"]) <= 0:
logger.error(
"No calibrator sources have been detected! Stopping pipeline.")
raise RuntimeError("No calibrator sources have been detected!")
# Amplitude calibration
if parms["doAmpCal"] and not check:
logger.info("--> Amplitude calibration (doAmpCal)")
plotFile = project + "_" + session + "_" + band + ".AmpCal.ps"
retCode = VLBAAmpCal(uv,
err,
calSou=parms["contCals"],
CalModel=parms["contCalModel"],
doCalib=2,
flagVer=2,
doBand=1,
minSumCC=parms["minFluxASC"],
refAnt=goodCal["bestRef"],
solInt=parms["solAInt"],
smoTimeA=2.0,
smoTimeP=1. / 60.,
doSNPlot=parms["doSNPlot"],
plotFile=plotFile,
nThreads=nThreads,
noScrat=noScrat,
logfile=logFile,
check=check,
debug=debug)
if retCode != 0:
raise RuntimeError("Error in amplitude calibration")
VLBASaveOutFiles() # Save plot file in Outfiles
# Rewrite contCals pickle file because contCals may have been updated
SaveObject(parms["contCals"], OKCalPicklefile, True)
if len(parms["contCals"]) <= 0:
logger.error(
"No calibrator sources have been detected! Stopping pipeline.")
raise RuntimeError("No calibrator sources have been detected!")
# Calibrate and average data
if parms["doCalAvg"]:
logger.info("--> Calibration and average data (doCalAvg)")
retCode = VLBACalAvg (uv, parms["avgClass"], seq, parms["CalAvgTime"], err, \
flagVer=2, doCalib=2, gainUse=0, doBand=1, BPVer=1, \
BIF=parms["CABIF"], EIF=parms["CAEIF"], \
BChan=parms["CABChan"], EChan=parms["CAEChan"], \
chAvg=parms["chAvg"], avgFreq=parms["avgFreq"], Compress=Compress, \
logfile=logFile, check=check, debug=debug)
if retCode != 0:
raise RuntimeError("Error in CalAvg")
# image targets phase only self-cal - NOTE: actually A&P
if parms["doImgTarget"] and not check:
logger.info("--> Image targets (doImgTargets)")
if not uvc:
# Get calibrated/averaged data
Aname = VLBAAIPSName(project, session)
uvname = project + "_" + session + "_" + band + "_Cal"
uvc = UV.newPAUV(uvname, Aname, parms["avgClass"], disk, seq, True,
err)
if err.isErr:
OErr.printErrMsg(err, "Error creating cal/avg AIPS data")
logger.debug("parms['targets'] = " + str(parms["targets"]))
VLBAImageCals(uv,
err,
Sources=parms["targets"],
seq=seq,
sclass=parms["outTclass"],
doCalib=2,
flagVer=2,
doBand=1,
FOV=parms["FOV"],
Robust=parms["Robust"],
maxPSCLoop=parms["maxPSCLoop"],
minFluxPSC=parms["minFluxPSC"],
solPInt=parms["solPInt"],
solMode=parms["solMode"],
maxASCLoop=parms["maxASCLoop"],
minFluxASC=parms["minFluxASC"],
solAInt=parms["solAInt"],
avgPol=parms["avgPol"],
avgIF=parms["avgIF"],
minSNR=parms["minSNR"],
refAnt=goodCal["bestRef"],
nThreads=nThreads,
noScrat=noScrat,
logfile=logFile,
check=check,
debug=debug)
# Rewrite targets pickle file because targets may have been updated
SaveObject(parms["targets"], targetsPicklefile, True)
if len(parms["targets"]) <= 0:
logger.error(
"No target sources have been detected! Stopping pipeline.")
raise RuntimeError("No target sources have been detected!")
# Phase calibration using target models
if parms["doPhaseCal"]:
logger.info("--> Phase calibration using target models (doPhaseCal)")
if not uvc:
# Get calibrated/averaged data
Aname = VLBAAIPSName(project, session)
uvname = project + "_" + session + "_" + band + "_Cal"
uvc = UV.newPAUV(uvname, Aname, parms["avgClass"], disk, seq, True,
err)
if err.isErr:
OErr.printErrMsg(err, "Error creating cal/avg AIPS data")
parms["targetModel"] = VLBAImageModel(parms["targets"],
parms["outTclass"], disk, seq,
err)
plotFile = project + "_" + session + "_" + band + ".PhaseCal.ps"
retCode = VLBAPhaseCal(uvc, err, calSou=parms["targets"], CalModel=parms["targetModel"], \
doCalib=-1, flagVer=0, doBand=-1, \
refAnt=goodCal["bestRef"], solInt=parms["manPCsolInt"], \
doSNPlot=parms["doSNPlot"], plotFile=plotFile, \
nThreads=nThreads, noScrat=noScrat, logfile=logFile, check=check, debug=debug)
if retCode != 0:
raise RuntimeError("Error in phase calibration")
VLBASaveOutFiles() # Save plot file in Outfiles
# Rewrite targets pickle file because targets may have been updated
SaveObject(parms["targets"], targetsPicklefile, True)
if len(parms["targets"]) <= 0:
logger.error(
"No target sources have been detected! Stopping pipeline.")
raise RuntimeError("No target sources have been detected!")
# Instrumental polarization calibration
if parms["doInstPol"]:
logger.info("--> Instrumental polarization calibration (doInstPol)")
# calibrators defaults to strong calibrator list
if not parms["instPolCal"]:
instPolCal = contCals
else:
instPolCal = parms["instPolCal"]
if not uvc:
# Get calibrated/averaged data
Aname = VLBAAIPSName(project, session)
uvname = project + "_" + session + "_" + band + "_Cal"
uvc = UV.newPAUV(uvname, Aname, parms["avgClass"], disk, seq, True,
err)
if err.isErr:
OErr.printErrMsg(err, "Error creating cal/avg AIPS data")
retCode = VLBAPolCal(uvc, instPolCal, err, \
doCalib=2, flagVer=0, doBand=-1, doSetJy=True, \
refAnt=goodCal["bestRef"], solInt=2.0, \
noScrat=noScrat, logfile=logFile, check=check, debug=debug)
if retCode != 0:
raise RuntimeError("Error in instrumental poln calibration")
# RL Phase (EVPA) calibration as BP table
if parms["doRLCal"] and parms["RLCal"]:
logger.info("--> RL phase calibration (doRLCal)")
if not uvc:
# Get calibrated/averaged data
Aname = VLBAAIPSName(project, session)
uvname = project + "_" + session + "_" + band + "_Cal"
uvc = UV.newPAUV(uvname, Aname, parms["avgClass"], disk, seq, True,
err)
if err.isErr:
OErr.printErrMsg(err, "Error creating cal/avg AIPS data")
retCode = VLBARLCal2(uvc, err, RLPCal=parms["RLCal"], \
doCalib=2, gainUse=2, flagVer=0, doBand=-1, doPol=True, \
refAnt=goodCal["bestRef"], niter=300, FOV=0.02/3600.0, \
nThreads=nThreads, noScrat=noScrat, logfile=logFile, check=check, debug=debug)
if retCode != 0:
raise RuntimeError("Error in RL phase calibration")
# image targets possible with Stokes I(QU)
if parms["doImgFullTarget"]:
logger.info("--> Image targets (doImgFullTargets)")
if not uvc:
# Get calibrated/averaged data
Aname = VLBAAIPSName(project, session)
uvname = project + "_" + session + "_" + band + "_Cal"
uvc = UV.newPAUV(uvname, Aname, parms["avgClass"], disk, seq, True,
err)
if err.isErr:
OErr.printErrMsg(err, "Error creating cal/avg AIPS data")
VLBAImageTargets(uvc,
err,
Sources=parms["targets"],
seq=seq,
sclass=parms["outIclass"],
doCalib=2,
flagVer=0,
doBand=-1,
Stokes=parms["Stokes"],
FOV=parms["FOV"],
Robust=parms["Robust"],
maxPSCLoop=2,
minFluxPSC=parms["minFluxPSC"],
solPInt=parms["solPInt"],
solMode="P",
maxASCLoop=parms["maxASCLoop"],
minFluxASC=parms["minFluxASC"],
solAInt=parms["solAInt"],
avgPol=parms["avgPol"],
avgIF=parms["avgIF"],
minSNR=parms["minSNR"],
refAnt=goodCal["bestRef"],
nTaper=parms["nTaper"],
Tapers=parms["Tapers"],
do3D=parms["do3D"],
nThreads=nThreads,
noScrat=noScrat,
logfile=logFile,
check=check,
debug=debug)
# Save UV data?
if parms["doSaveUV"] and (not check):
logger.info("--> Save UV data (doSaveUV)")
mess = "Write calibrated and averaged UV data to disk"
printMess(mess, logFile)
# Get calibrated/averaged data
if not uvc:
Aname = VLBAAIPSName(project, session)
uvname = project + "_" + session + "_" + band + "_Cal"
uvc = UV.newPAUV(uvname, Aname, parms["avgClass"], disk, seq, True,
err)
if err.isErr:
OErr.printErrMsg(err, "Error creating cal/avg AIPS data")
# Write
filename = project + session + band + "CalAvg.uvtab"
fuv = VLBAUVFITS(uvc, filename, 0, err, compress=Compress)
VLBAAddOutFile(filename, 'project', "Calibrated Averaged UV data")
# Save list of output files
VLBASaveOutFiles()
# Save UV data tables?
if parms["doSaveTab"] and (not check):
logger.info("--> Save UV data tables (doSaveTab)")
mess = "Write UV data tables to disk."
printMess(mess, logFile)
filename = project + session + band + "CalTab.uvtab"
fuv = VLBAUVFITSTab(uv, filename, 0, err)
VLBAAddOutFile(filename, 'project', "Calibrated AIPS tables")
# Save list of output files
VLBASaveOutFiles()
# Imaging results
outDisk = 0
if parms["doSaveImg"]:
logger.info("--> Save images (doSaveImg)")
mess = "Write images to disk."
printMess(mess, logFile)
# How many Stokes images
nstok = len(parms["Stokes"])
# Targets
logger.debug("parms['targets'] = " + str(parms['targets']))
for target in parms["targets"]:
if not check:
# intermediate images
oclass = parms["outTclass"]
x = Image.newPAImage("out", target, oclass, disk, seq, True,
err)
if (not x.exist):
print(target, "image not found. Skipping.")
continue
outfile = project + session + band + target + "." + oclass + ".fits"
mess = "Write Intermediate target " + outfile + " on disk " + str(
outDisk)
VLBAAddOutFile(outfile, target, 'Intermediate target image')
printMess(mess, logFile)
xf = VLBAImFITS(x, outfile, outDisk, err, fract=0.1)
# Save list of output files
VLBASaveOutFiles()
# Statistics
zz = imstat(x, err, logfile=logFile)
del x, xf
# Final images
for istok in range(0, nstok):
oclass = parms["Stokes"][istok:istok +
1] + parms["outIclass"][1:]
x = Image.newPAImage("out", target, oclass, disk, seq,
True, err)
outfile = project + session + band + target + "." + oclass + ".fits"
logger.info("Write " + outfile + " on disk " +
str(outDisk))
xf = VLBAImFITS(x, outfile, outDisk, err, fract=0.1)
VLBAAddOutFile(outfile, target, 'Image')
logger.info("Writing file " + outfile)
# Statistics
zz = imstat(x, err, logfile=logFile)
del x, xf
# Save list of output files
VLBASaveOutFiles()
# Calibrators
for target in parms["contCals"]:
if not check:
oclass = parms["outCclass"]
x = Image.newPAImage("out", target, oclass, disk, seq, True,
err)
if (not x.exist):
print(target, "image not found. Skipping.")
continue
outfile = project + session + band + target + "." + oclass + ".fits"
mess = "Write Calibrator " + outfile + " on disk " + str(
outDisk)
printMess(mess, logFile)
xf = VLBAImFITS(x, outfile, outDisk, err, fract=0.1)
VLBAAddOutFile(outfile, target, 'Calibrator Image')
# Statistics
zz = imstat(x, err, logfile=logFile)
del x, xf
# Save list of output files
VLBASaveOutFiles()
# end writing images loop
# Contour plots
if parms["doKntrPlots"]:
logger.info("--> Contour plots (doKntrPlots)")
VLBAKntrPlots(err,
imName=parms["targets"],
project=project,
session=session,
band=band,
disk=disk,
debug=debug)
# Save list of output files
VLBASaveOutFiles()
elif debug:
print("Not creating contour plots ( doKntrPlots = ",
parms["doKntrPlots"], " )")
# Source uv plane diagnostic plots
if parms["doDiagPlots"]:
logger.info("--> Diagnostic plots (doDiagPlots)")
# Get the highest number avgClass catalog file
Aname = VLBAAIPSName(project, session)
uvc = None
if not check:
uvname = project + "_" + session + "_" + band + "_Cal"
uvc = UV.newPAUV(uvname, Aname, parms["avgClass"], disk, seq, True,
err)
VLBADiagPlots( uvc, err, cleanUp=parms["doCleanup"], \
project=project, session=session, band=band, \
logfile=logFile, check=check, debug=debug )
# Save list of output files
VLBASaveOutFiles()
elif debug:
print("Not creating diagnostic plots ( doDiagPlots = ",
parms["doDiagPlots"], " )")
# Save metadata
srcMetadata = None
projMetadata = None
if parms["doMetadata"]:
logger.info("--> Save metadata (doMetadata)")
if not uvc:
# Get calibrated/averaged data
Aname = VLBAAIPSName(project, session)
uvname = project + "_" + session + "_" + band + "_Cal"
uvc = UV.newPAUV(uvname, Aname, parms["avgClass"], disk, seq, True,
err)
if err.isErr:
OErr.printErrMsg(err, "Error creating cal/avg AIPS data")
# Get source metadata; save to pickle file
srcMetadata = VLBASrcMetadata(uvc,
err,
Sources=parms["targets"],
seq=seq,
sclass=parms["outIclass"],
Stokes=parms["Stokes"],
logfile=logFile,
check=check,
debug=debug)
picklefile = project + "_" + session + "_" + band + "SrcReport.pickle"
SaveObject(srcMetadata, picklefile, True)
VLBAAddOutFile(picklefile, 'project', 'All source metadata')
# Get project metadata; save to pickle file
projMetadata = VLBAProjMetadata(uvc,
AIPS_VERSION,
err,
contCals=parms["contCals"],
goodCal=goodCal,
project=project,
session=session,
band=band,
dataInUVF=dataInUVF,
archFileID=archFileID)
picklefile = project + "_" + session + "_" + band + "ProjReport.pickle"
SaveObject(projMetadata, picklefile, True)
VLBAAddOutFile(picklefile, 'project', 'Project metadata')
# Write report
if parms["doHTML"]:
logger.info("--> Write HTML report (doHTML)")
VLBAHTMLReport( projMetadata, srcMetadata, \
outfile=project+"_"+session+"_"+band+"report.html", \
logFile=logFile )
# Write VOTable
if parms["doVOTable"]:
logger.info("--> Write VOTable (doVOTable)")
VLBAAddOutFile('VOTable.xml', 'project', 'VOTable report')
VLBAWriteVOTable(projMetadata, srcMetadata, filename='VOTable.xml')
# Save list of output files
VLBASaveOutFiles()
# Cleanup - delete AIPS files
if parms["doCleanup"] and (not check):
logger.info("--> Clean up (doCleanup)")
# Delete target images
# How many Stokes images
nstok = len(parms["Stokes"])
for istok in range(0, nstok):
oclass = parms["Stokes"][istok:istok + 1] + parms["outIclass"][1:]
AllDest(err, disk=disk, Aseq=seq, Aclass=oclass)
# delete Calibrator images
AllDest(err, disk=disk, Aseq=seq, Aclass=parms["outCclass"])
# Delete intermediate target images
AllDest(err, disk=disk, Aseq=seq, Aclass=parms["outTclass"])
OErr.printErrMsg(err, "Deleting AIPS images")
# Delete UV data
uv.Zap(err)
# Zap calibrated/averaged data
if not uvc:
Aname = VLBAAIPSName(project, session)
uvc = UV.newPAUV("AIPS UV DATA", Aname, parms["avgClass"], disk,
seq, True, err)
if err.isErr:
OErr.printErrMsg(err, "Error creating cal/avg AIPS data")
uvc.Zap(err)
OErr.printErrMsg(err, "Writing output/cleanup")
# Shutdown
mess = "Finished project " + project
printMess(mess, logFile)
OErr.printErr(err)
OSystem.Shutdown(ObitSys)
def pipeline(aipsSetup, parmFile):
"""
ALMA Continuum pipeline.
* *aipsSetup* = AIPS setup file
* *parmFile* = pipeline input parameters file
"""
############################# Initialize OBIT ##########################################
noScrat = []
exec(open(aipsSetup).read())
ALMAAddOutFile(aipsSetup, 'project', "Obit's AIPS setup file")
############################# Default parameters ##########################################
# Initialize parameters
parms = ALMAInitContParms()
############################# Set Project Processing parameters ##################
print "parmFile", parmFile
exec(open(parmFile).read())
ALMAAddOutFile(parmFile, 'project', 'Pipeline input parameters')
# frequency/configuration dependent default parameters
ALMAInitContFQParms(parms)
# General data parameters
band = parms["band"] # Observing band
dataClass = ("UVDa" + band)[0:6] # AIPS class of raw uv data
project = parms["project"][
0:12] # Project name (12 char or less, used as AIPS Name)
session = parms["session"] # Project session code
################################## Process #####################################
fileRoot = parms["project"] + "_" + parms["session"] + "_" + parms[
"band"] # root of file name
logFile = fileRoot + ".log" # Processing log file
uv = None
uvc = None
avgClass = ("UVAv" + band)[0:6] # Averaged data AIPS class
outIClass = parms["outIClass"] # image AIPS class
outCClass = parms["outCClass"] # image AIPS class
# Load the outputs pickle jar
ALMAFetchOutFiles()
# Logging directly to logFile
OErr.PInit(err, parms["prtLv"], logFile)
OSystem.PAllowThreads(nThreads) # Allow threads in Obit/oython
retCode = 0
ALMAAddOutFile(logFile, 'project', 'Pipeline log file')
mess = "Start project "+parms["project"]+" session "+parms["session"]+\
" "+parms["band"]+" Band"+" AIPS user no. "+str(AIPS.userno)+\
", ALMA Max. baseline "+str(parms["ALMAMaxBl"])
printMess(mess, logFile)
if debug:
pydoc.ttypager = pydoc.plainpager # don't page task input displays
mess = "Using Debug mode "
printMess(mess, logFile)
if check:
mess = "Only checking script"
printMess(mess, logFile)
# Log parameters
if parms['doLogParms']:
printMess("Parameter settings", logFile)
for p in parms:
mess = " " + p + ": " + str(parms[p])
printMess(mess, logFile)
# Save parameters to pickle jar, manifest
ParmsPicklefile = project + "_" + session + "_" + band + ".Parms.pickle" # Where results saved
SaveObject(parms, ParmsPicklefile, True)
ALMAAddOutFile(ParmsPicklefile, 'project', 'Processing parameters used')
# Are we going to be doing Hanning?
if parms["doHann"]:
loadClass = parms["band"] + "Raw"
else:
loadClass = dataClass
# Load Data from Archive directory
if parms["doLoadArchive"]:
uv = ALMAUVLoadArch(parms["archRoots"], ALMAAIPSName(project, session), loadClass, disk, parms["seq"], err, \
selConfig=parms["selConfig"], selBand=parms["selBand"], selChan=parms["selChan"], \
selChBW=parms["selChBW"], selNIF=parms["selNIF"], calInt=parms["calInt"], \
logfile=logFile, Compress=parms["Compress"], check=check, debug=debug)
if uv == None and not check:
raise RuntimeError, "Cannot load " + parms["archRoots"]
# Hanning
if parms["doHann"]:
# Set uv if not done
if uv == None and not check:
uv = UV.newPAUV("AIPS UV DATA", ALMAAIPSName(project, session),
loadClass[0:6], disk, parms["seq"], True, err)
if err.isErr:
OErr.printErrMsg(err, "Error creating AIPS data")
uv = ALMAHann(uv, ALMAAIPSName(project, session), dataClass, disk, parms["seq"], err, \
doDescm=parms["doDescm"], logfile=logFile, check=check, debug=debug)
if uv == None and not check:
raise RuntimeError, "Cannot Hann data "
# Set uv if not done
if uv == None and not check:
uv = UV.newPAUV("AIPS UV DATA", ALMAAIPSName(project, session), dataClass[0:6], \
disk, parms["seq"], True, err)
if err.isErr:
OErr.printErrMsg(err, "Error creating AIPS data")
# Clear any old calibration/editing
if parms["doClearTab"]:
mess = "Clear previous calibration"
printMess(mess, logFile)
ALMAClearCal(uv,
err,
doGain=parms["doClearGain"],
doFlag=parms["doClearFlag"],
doBP=parms["doClearBP"],
check=check)
OErr.printErrMsg(err, "Error resetting calibration")
# Copy FG 1 to FG 2
if parms["doCopyFG"]:
mess = "Copy FG 1 to FG 2"
printMess(mess, logFile)
retCode = ALMACopyFG(uv,
err,
logfile=logFile,
check=check,
debug=debug)
if retCode != 0:
raise RuntimeError, "Error Copying FG table"
# Drop end channels of spectra? Only if new FG 2
if parms["doCopyFG"] and (parms["BChDrop"] > 0) or (parms["EChDrop"] > 0):
# Channels based on original number, reduced if Hanning
nchan = uv.Desc.Dict["inaxes"][uv.Desc.Dict["jlocf"]]
fact = max(1, parms["selChan"] / nchan) # Hanning reduction factor
BChDrop = parms["BChDrop"] / fact
EChDrop = parms["EChDrop"] / fact
mess = "Trim %d channels from start and %d from end of each spectrum" % (
BChDrop, EChDrop)
printMess(mess, logFile)
retCode = ALMADropChan (uv, BChDrop, EChDrop, err, flagVer=parms["editFG"], \
logfile=logFile, check=check, debug=debug)
if retCode != 0:
raise RuntimeError, "Error Copying FG table"
# Get list of source in data
AllSource = ALMAAllSource(uv,
err,
logfile=logFile,
check=check,
debug=debug)
# Edit source lists to remove sources not present
for c in parms["PCInsCals"]:
if c not in AllSource:
parms["PCInsCals"].remove(c)
for t in parms["targets"]:
if t not in AllSource:
parms["targets"].remove(t)
if parms["XYGainSource"] not in AllSource:
parms["XYGainSource"] = None
if parms["XYDelaySource"] not in AllSource:
parms["XYDelaySource"] = None
for c in parms["DCals"]:
if c['Source'] not in AllSource:
c['Source'] = None
for c in parms["BPCals"]:
if c['Source'] not in AllSource:
c['Source'] = None
for c in parms["PCals"]:
if c['Source'] not in AllSource:
c['Source'] = None
for c in parms["ACals"]:
if c['Source'] not in AllSource:
c['Source'] = None
# Special editing
if parms["doEditList"] and not check:
mess = "Special editing"
printMess(mess, logFile)
for edt in parms["editList"]:
UV.PFlag(uv,err,timeRange=[dhms2day(edt["timer"][0]),dhms2day(edt["timer"][1])], \
flagVer=parms["editFG"], Ants=edt["Ant"], Chans=edt["Chans"], IFs=edt["IFs"], \
Stokes=edt["Stokes"], Reason=edt["Reason"])
OErr.printErrMsg(err, "Error Flagging")
# Quack to remove data from start and end of each scan
if parms["doQuack"]:
retCode = ALMAQuack (uv, err, begDrop=parms["quackBegDrop"], endDrop=parms["quackEndDrop"], \
Reason=parms["quackReason"], \
logfile=logFile, check=check, debug=debug)
if retCode != 0:
raise RuntimeError, "Error Quacking data"
# Flag antennas shadowed by others?
if parms["doShad"]:
retCode = ALMAShadow (uv, err, shadBl=parms["shadBl"], \
logfile=logFile, check=check, debug=debug)
if retCode != 0:
raise RuntimeError, "Error Shadow flagging data"
# Apply online calibration
if parms["doOnlineCal"]:
retCode = ALMAOnlineCal(uv,
err,
logfile=logFile,
check=check,
debug=debug)
if retCode != 0:
raise RuntimeError, "Error applying online calibration"
# Median window time editing, for RFI impulsive in time
if parms["doMedn"]:
mess = "Median window time editing, for RFI impulsive in time:"
printMess(mess, logFile)
retCode = ALMAMedianFlag (uv, " ", err, noScrat=noScrat, nThreads=nThreads, \
avgTime=parms["avgTime"], avgFreq=parms["avgFreq"], chAvg= parms["chAvg"], \
timeWind=parms["timeWind"], flagVer=2,flagSig=parms["mednSigma"], \
logfile=logFile, check=check, debug=False)
if retCode != 0:
raise RuntimeError, "Error in MednFlag"
# Median window frequency editing, for RFI impulsive in frequency
if parms["doFD1"]:
mess = "Median window frequency editing, for RFI impulsive in frequency:"
printMess(mess, logFile)
retCode = ALMAAutoFlag (uv, " ", err, flagVer=2, doCalib=-1, doBand=-1, \
timeAvg=parms["FD1TimeAvg"], \
doFD=True, FDmaxAmp=1.0e20, FDmaxV=1.0e20, FDwidMW=parms["FD1widMW"], \
FDmaxRMS=[1.0e20,0.1], FDmaxRes=parms["FD1maxRes"], \
FDmaxResBL= parms["FD1maxRes"], \
nThreads=nThreads, logfile=logFile, check=check, debug=debug)
if retCode != 0:
raise RuntimeError, "Error in AutoFlag"
# RMS/Mean editing for calibrators
if parms["doRMSAvg"]:
mess = "RMS/Mean editing for calibrators:"
printMess(mess, logFile)
clist = ALMACombineCals(parms["ACals"], parms["PCals"],
parms["DCals"]) # Calibrator list
retCode = ALMAAutoFlag (uv, clist, err, flagVer=2, doCalib=-1, doBand=-1, \
RMSAvg=parms["RMSAvg"], timeAvg=parms["RMSTimeAvg"], \
nThreads=nThreads, logfile=logFile, check=check, debug=debug)
if retCode != 0:
raise RuntimeError, "Error in AutoFlag"
# Need to find a reference antenna? See if we have saved it?
if (parms["refAnt"] <= 0):
refAnt = FetchObject(project + "_" + session + "_" + band +
".refAnt.pickle")
if refAnt:
parms["refAnt"] = refAnt
# Use bandpass calibrator and center half of each spectrum
if parms["refAnt"] <= 0:
mess = "Find best reference antenna: run Calib on BP Cal(s) "
printMess(mess, logFile)
parms["refAnt"] = ALMAGetRefAnt(uv, parms["BPCals"], err, flagVer=2, \
solInt=parms["bpsolint1"], nThreads=nThreads, \
logfile=logFile, check=check, debug=debug)
if err.isErr:
raise RuntimeError, "Error finding reference antenna"
if parms["refAnts"][0] <= 0:
parms["refAnts"][0] = parms["refAnt"]
mess = "Picked reference antenna " + str(parms["refAnt"])
printMess(mess, logFile)
# Save it
ParmsPicklefile = project + "_" + session + "_" + band + ".Parms.pickle" # Where results saved
SaveObject(parms, ParmsPicklefile, True)
refAntPicklefile = project + "_" + session + "_" + band + ".refAnt.pickle" # Where results saved
SaveObject(parms["refAnt"], refAntPicklefile, True)
# Plot Raw, edited data?
if parms["doRawSpecPlot"] and parms["plotSource"]:
mess = "Raw Spectral plot for: " + parms["plotSource"]
printMess(mess, logFile)
plotFile = "./" + fileRoot + "RawSpec.ps"
retCode = ALMASpectrum(uv, parms["plotSource"], parms["plotTime"], plotFile, parms["refAnt"], err, \
flagVer=2, Stokes=["XX","YY"], doband=-1, \
check=check, debug=debug, logfile=logFile )
if retCode != 0:
raise RuntimeError, "Error in Plotting spectrum"
ALMAAddOutFile(plotFile, 'project', 'Pipeline log file')
# delay calibration
if parms["doDelayCal"] and parms["DCals"] and not check:
plotFile = "./" + fileRoot + "DelayCal.ps"
retCode = ALMADelayCal(uv, parms["DCals"], err, \
BChan=parms["delayBChan"], EChan=parms["delayEChan"], \
doCalib=2, flagVer=2, doBand=-1, \
solInt=parms["delaySolInt"], smoTime=parms["delaySmoo"], \
refAnts=[parms["refAnt"]], doTwo=parms["doTwo"],
doZeroPhs=parms["delayZeroPhs"], \
doPlot=parms["doSNPlot"], plotFile=plotFile, \
nThreads=nThreads, noScrat=noScrat, \
logfile=logFile, check=check, debug=debug)
if retCode != 0:
raise RuntimeError, "Error in delay calibration"
# Plot corrected data?
if parms["doSpecPlot"] and parms["plotSource"]:
plotFile = "./" + fileRoot + "DelaySpec.ps"
retCode = ALMASpectrum(uv, parms["plotSource"], parms["plotTime"], \
plotFile, parms["refAnt"], err, \
flagVer=2, Stokes=["XX","YY"], doband=-1, \
check=check, debug=debug, logfile=logFile )
if retCode != 0:
raise RuntimeError, "Error in Plotting spectrum"
# Bandpass calibration
if parms["doBPCal"] and parms["BPCals"]:
plotFile = "./" + fileRoot + "BPCal.ps"
retCode = ALMABPCal(uv, parms["BPCals"], err, noScrat=noScrat, solInt1=parms["bpsolint1"], \
solInt2=parms["bpsolint2"], solMode=parms["bpsolMode"], \
BChan1=parms["bpBChan1"], EChan1=parms["bpEChan1"], \
BChan2=parms["bpBChan2"], EChan2=parms["bpEChan2"], ChWid2=parms["bpChWid2"], \
doCenter1=parms["bpDoCenter1"], refAnt=parms["refAnt"], \
UVRange=parms["bpUVRange"], doCalib=2, gainUse=0, flagVer=2, doPlot=False, \
doAmpEdit=parms["doAmpEdit"], ampSigma=parms["ampSigma"], \
doBPPlot=parms["doBPPlot"], plotBPFile=plotFile, \
nThreads=nThreads, logfile=logFile, check=check, debug=debug)
if retCode != 0:
raise RuntimeError, "Error in Bandpass calibration"
# Plot corrected data?
if parms["doSpecPlot"] and parms["plotSource"]:
plotFile = "./" + fileRoot + "BPSpec.ps"
retCode = ALMASpectrum(uv, parms["plotSource"], parms["plotTime"], plotFile, \
parms["refAnt"], err, flagVer=2, Stokes=["XX","YY"], doband=1, \
check=check, debug=debug, logfile=logFile )
if retCode != 0:
raise RuntimeError, "Error in Plotting spectrum"
# set X/Y gains and initial calibration
if parms["doXYFixGain"] and parms["XYGainSource"] and not check:
mess = "Fix X/Y gain ratios"
printMess(mess, logFile)
retCode = ALMAXYGain(uv, err, \
XYCal=parms["XYGainSource"],timerange=parms["XYGainTime"], \
doCalib=2, gainUse=0, doBand=1, flagVer=2, refAnt=parms["refAnt"], \
nThreads=nThreads, noScrat=noScrat, logfile=logFile, \
check=check, debug=debug)
if retCode != 0:
raise RuntimeError, "Error in X-Y gain fix"
# Self calibrate calibrators
if parms["doImgCal"] and not check:
mess = "SelfCalibrate/Image calibrators"
printMess(mess, logFile)
src = ALMACombineCals(parms["ACals"], parms["PCals"],
parms["APCals"]) # Source list
ALMAImageCals(uv, err, Sources=src, seq=parms["seq"], \
sclass=parms["outCClass"], doCalib=2, flagVer=2, doBand=1, FOV=parms["CalFOV"], \
maxPSCLoop=parms["maxPSCLoop"], minFluxPSC=parms["minFluxPSC"], solPInt=parms["solPInt"], \
maxASCLoop=parms["maxASCLoop"], minFluxASC=parms["minFluxASC"],\
solAInt=parms["solAInt"], avgPol=parms["avgPol"], Niter=1000, \
avgIF=parms["avgIF"], minSNR=parms["minSNR"],\
refAnt=parms["refAnt"], nThreads=nThreads, noScrat=noScrat,\
logfile=logFile, check=check, debug=debug)
# Self calibrated models now available
ALMAImageModel(parms["ACals"], parms["outCClass"], disk, parms["seq"], err)
ALMAImageModel(parms["PCals"], parms["outCClass"], disk, parms["seq"], err)
ALMAImageModel(parms["APCals"], parms["outCClass"], disk, parms["seq"],
err)
# Phase Calibrate
if parms["doPhaseCal"]:
plotFile = "./" + fileRoot + "PhaseCal.ps"
retCode = ALMAPhaseCal (uv, parms["PCals"], err, ACals=parms["ACals"], \
doCalib=2, doBand=1, BPVer=1, flagVer=2, refAnt=parms["refAnt"], \
BChan=parms["ampBChan"], EChan=parms["ampEChan"], \
solInt=parms["solPInt"], avgIF = parms['avgIF'], \
ampScalar=parms["ampScalar"], doPlot=parms["doSNPlot"], plotFile=plotFile, \
nThreads=nThreads, noScrat=noScrat, logfile=logFile, check=check, debug=debug)
if retCode != 0:
raise RuntimeError, "Error calibrating"
# Amp & phase Calibrate
if parms["doAmpPhaseCal"]:
plotFile = "./" + fileRoot + "APCal.ps"
retCode = ALMACalAP (uv, [], parms["ACals"], err, PCals=parms["APCals"], \
doCalib=2, doBand=1, BPVer=1, flagVer=2, \
BChan=parms["ampBChan"], EChan=parms["ampEChan"], \
solInt=parms["solAInt"], solSmo=parms["solSmo"], ampScalar=parms["ampScalar"], \
doAmpEdit=parms["doAmpEdit"], ampSigma=parms["ampSigma"], \
ampEditFG=parms["ampEditFG"], \
doPlot=parms["doSNPlot"], plotFile=plotFile, refAnt=parms["refAnt"], \
nThreads=nThreads, noScrat=noScrat, logfile=logFile, check=check, debug=debug)
if retCode != 0:
raise RuntimeError, "Error calibrating"
# More editing
if parms["doAutoFlag"]:
mess = "Post calibration editing:"
printMess(mess, logFile)
clist = []
retCode = ALMAAutoFlag (uv, clist, err, flagVer=2, \
doCalib=2, gainUse=0, doBand=1, BPVer=1, \
IClip=parms["IClip"], minAmp=parms["minAmp"], timeAvg=parms["timeAvg"], \
doFD=parms["doAFFD"], FDmaxAmp=parms["FDmaxAmp"], FDmaxV=parms["FDmaxV"], \
FDwidMW=parms["FDwidMW"], FDmaxRMS=parms["FDmaxRMS"], \
FDmaxRes=parms["FDmaxRes"], FDmaxResBL=parms["FDmaxResBL"], \
FDbaseSel=parms["FDbaseSel"], \
nThreads=nThreads, logfile=logFile, check=check, debug=debug)
if retCode != 0:
raise RuntimeError, "Error in AutoFlag"
# Calibrate and average data
if parms["doCalAvg"]:
retCode = ALMACalAvg (uv, avgClass, parms["seq"], parms["CalAvgTime"], err, \
flagVer=2, doCalib=2, gainUse=0, doBand=1, BPVer=1, doPol=False, \
avgFreq=parms["CAavgFreq"], chAvg=parms["CAchAvg"], \
BChan=parms["CABChan"], EChan=parms["CAEChan"], \
BIF=parms["CABIF"], EIF=parms["CAEIF"], Compress=parms["Compress"], \
nThreads=nThreads, logfile=logFile, check=check, debug=debug)
if retCode != 0:
raise RuntimeError, "Error in CalAvg"
# Get calibrated/averaged data
if not check:
try:
uv = UV.newPAUV("AIPS UV DATA", ALMAAIPSName(project, session), avgClass[0:6], \
disk, parms["seq"], True, err)
except Exception, exception:
mess = "No Averaged/calibrated datafile"
printMess(mess, logFile)
return
