def UVMakeIF(outUV, nIF, err, solInt=10.): """ Change number of IFs from 1 to nIF Operation done in place * outUV = output Obit UV object Only really works for AIPS * nIF = number of desired output IFs MUST be the same number of channels per IF * err = Obit error/message stack * solInt = Solution interval for remade CL table. """ ################################################################ # Checks if not UV.PIsA(outUV): raise TypeError("outUV MUST be a defined Python Obit UV") # Input can have 1 or no IFs defined jlocif = outUV.Desc.Dict["jlocif"] if jlocif >= 0 and outUV.Desc.Dict["inaxes"][jlocif] > 1: raise RuntimeError("Input UV has excessive IFs already:" \ +str(outUV.Desc.Dict["inaxes"][jlocif])) # Check number of requested IFs if nIF < 2: raise RuntimeError("Too few output IFs requested: " + str(nIF)) # Must be an even number of channels per output IF jlocf = outUV.Desc.Dict["jlocf"] nchan = outUV.Desc.Dict["inaxes"][jlocf] if (nchan % nIF) != 0: raise RuntimeError("Unequal numbers of channels per " + str(nIF) + " IFs") # Patch UV Descriptor DescMakeIF(outUV, nIF, err) # Convert FQ Table UpdateFQ2(outUV, nIF, err) # Convert AN Table maxant = UpdateAN2(outUV, nIF, err) # Convert SU Table UpdateSU2(outUV, nIF, err) # Regenerate CL table 1 - delete any old outUV.ZapTable("AIPS CL", -1, err) print('(Re) generate CL table') UV.PTableCLfromNX(outUV, maxant, err, calInt=solInt) # dummy FG 1 print('Dummy entry in Flag table 1') UV.PFlag(outUV, err, timeRange=[-10., -9.], Ants=[200, 200], Stokes='0000', Reason='Dummy') # Update outUV.UpdateDesc(err) OErr.printErrMsg(err, "Error updating output")
def WriteFGTable(outUV, katdata, meta, err): """ Get the flags from the h5 file and convert to FG table format. UNUSED- This implimentation is too slow! outUV = Obit UV object meta = dict with data meta data err = Python Obit Error/message stack to init """ ############################################################### # work out Start time in unix sec tm = katdata.timestamps[1:2] tx = time.gmtime(tm[0]) time0 = tm[0] - tx[3] * 3600.0 - tx[4] * 60.0 - tx[5] int_time = katdata.dump_period #Loop through scans in h5 file row = 0 for scan, state, target in katdata.scans(): name = target.name.replace(' ', '_') if state != 'track': continue tm = katdata.timestamps[:] nint = len(tm) el = target.azel(tm[int(nint / 2)])[1] * 180. / math.pi if el < 15.0: continue row += 1 flags = katdata.flags()[:] numflag = 0 for t, chan_corr in enumerate(flags): for c, chan in enumerate(chan_corr): cpflagged = [] for p, cp in enumerate(chan): #for cpaverage in meta['pairLookup']: flag = cp #numpy.any(chan[meta['pairLookup'][cpaverage]]) product = meta['products'][p] if product[0] == product[1]: continue if flag and (not product[0:2] in cpflagged): cpflagged.append(product[0:2]) numflag += 1 starttime = float( (tm[t] - time0 - (int_time / 2)) / 86400.0) endtime = float( (tm[t] - time0 + (int_time / 2)) / 86400.0) UV.PFlag(outUV,err,timeRange=[starttime,endtime], flagVer=1, \ Ants=[product[0],product[1]], \ Chans=[c+1,c+1], IFs=[1,1], Stokes='1111', Reason='Online flag') numvis = t * c * (p / meta["nstokes"]) msg = "Scan %4d %16s Online flags: %7s of %8s vis" % ( row, name, numflag, numvis) OErr.PLog(err, OErr.Info, msg) OErr.printErr(err)
def pipeline(scriptName, aipsSetup, parmFile): """ Linear (feed) Polarization Continuum pipeline. * *scriptName* = this script file name * *aipsSetup* = AIPS setup file * *parmFile* = pipeline input parameters file """ ############################# Initialize OBIT ########################################## noScrat = [] exec(open(aipsSetup).read()) EVLAAddOutFile(aipsSetup, 'project', "Obit's AIPS setup file") ############################# Default parameters ########################################## # Initialize parameters parms = EVLAInitContParms() # Linear feed specific parameters # Relative gain calibration parms["doXYRelGain"] = False # constrain relative X, Y gain amplitudes parms[ "doXYRelGain2"] = None # constrain relative X, Y gain amplitudes, 2nd pass parms["XYRelGainCal"] = None # X/Y gain calibrator structure parms["XYRelGainCalTime"] = [0., 0. ] # Time range for relative gain calibration ############################# Set Project Processing parameters ################## print("parmFile", parmFile) exec(open(parmFile).read()) EVLAAddOutFile(parmFile, 'project', 'Pipeline input parameters') # frequency/configuration dependent default parameters EVLAInitContFQParms(parms) # Linear feed specific: parms["avgPol"] = parms["doXYRelGain"] and (parms["XYRelGainCal"] != None ) # Fix relative X/Y gains? # General data parameters band = parms["band"] # Observing band dataClass = ("UVDa" + band)[0:6] # AIPS class of raw uv data project = parms["project"][ 0:12] # Project name (12 char or less, used as AIPS Name) session = parms["session"] # Project session code ################################## Process ##################################### fileRoot = parms["project"] + "_" + parms["session"] + "_" + parms[ "band"] # root of file name logFile = fileRoot + ".log" # Processing log file uv = None uvc = None avgClass = ("UVAv" + band)[0:6] # Averaged data AIPS class outIClass = parms["outIClass"] # image AIPS class # Load the outputs pickle jar EVLAFetchOutFiles() # Logging directly to logFile OErr.PInit(err, parms["prtLv"], logFile) OSystem.PAllowThreads(nThreads) # Allow threads in Obit/oython retCode = 0 EVLAAddOutFile(logFile, 'project', 'Pipeline log file') mess = "Start project "+parms["project"]+" session "+parms["session"]+\ " "+parms["band"]+" Band"+" AIPS user no. "+str(AIPS.userno)+\ ", EVLA configuration "+parms["VLACfg"] printMess(mess, logFile) if debug: pydoc.ttypager = pydoc.plainpager # don't page task input displays mess = "Using Debug mode " printMess(mess, logFile) if check: mess = "Only checking script" printMess(mess, logFile) # Log parameters printMess("Parameter settings", logFile) for p in parms: mess = " " + p + ": " + str(parms[p]) printMess(mess, logFile) # Save parameters to pickle jar, manifest ParmsPicklefile = project + "_" + session + "_" + band + ".Parms.pickle" # Where results saved SaveObject(parms, ParmsPicklefile, True) EVLAAddOutFile(ParmsPicklefile, 'project', 'Processing parameters used') # Are we going to be doing Hanning? if parms["doHann"]: loadClass = parms["band"] + "Raw" else: loadClass = dataClass # Load Data from Archive directory if parms["doLoadArchive"]: uv = EVLAUVLoadArch(parms["archRoot"], EVLAAIPSName(project, session), loadClass, disk, parms["seq"], err, \ selConfig=parms["selConfig"], doSwPwr=parms["doSwPwr"], \ selBand=parms["selBand"], selChan=parms["selChan"], \ selNIF=parms["selNIF"], calInt=parms["calInt"], \ logfile=logFile, Compress=parms["Compress"], check=check, debug=debug) if uv == None and not check: raise RuntimeError("Cannot load " + parms["DataRoot"]) # Hanning if parms["doHann"]: # Set uv if not done if uv == None and not check: uv = UV.newPAUV("AIPS UV DATA", EVLAAIPSName(project, session), loadClass[0:6], disk, parms["seq"], True, err) if err.isErr: OErr.printErrMsg(err, "Error creating AIPS data") uv = EVLAHann(uv, EVLAAIPSName(project, session), dataClass, disk, parms["seq"], err, \ doDescm=parms["doDescm"], logfile=logFile, check=check, debug=debug) if uv == None and not check: raise RuntimeError("Cannot Hann data ") # Set uv if not done if uv == None and not check: uv = UV.newPAUV("AIPS UV DATA", EVLAAIPSName(project, session), dataClass[0:6], \ disk, parms["seq"], True, err) if err.isErr: OErr.printErrMsg(err, "Error creating AIPS data") # Save file names in history EVLAScriptHistory(uv, scriptName, aipsSetup, parmFile, err) if err.isErr: OErr.printErrMsg(err, "Error writing file names to history") # Clear any old calibration/editing if parms["doClearTab"]: mess = "Clear previous calibration" printMess(mess, logFile) EVLAClearCal(uv, err, doGain=parms["doClearGain"], doFlag=parms["doClearFlag"], doBP=parms["doClearBP"], check=check) OErr.printErrMsg(err, "Error resetting calibration") # Copy FG 1 to FG 2 if parms["doCopyFG"]: mess = "Copy FG 1 to FG 2" printMess(mess, logFile) retCode = EVLACopyFG(uv, err, logfile=logFile, check=check, debug=debug) if retCode != 0: raise RuntimeError("Error Copying FG table") # Drop end channels of spectra? Only if new FG 2 if parms["doCopyFG"] and (parms["BChDrop"] > 0) or (parms["EChDrop"] > 0): # Channels based on original number, reduced if Hanning nchan = uv.Desc.Dict["inaxes"][uv.Desc.Dict["jlocf"]] fact = parms["selChan"] / nchan # Hanning reduction factor BChDrop = parms["BChDrop"] / fact EChDrop = parms["EChDrop"] / fact mess = "Trim %d channels from start and %d from end of each spectrum" % ( BChDrop, EChDrop) printMess(mess, logFile) retCode = EVLADropChan (uv, BChDrop, EChDrop, err, flagVer=parms["editFG"], \ logfile=logFile, check=check, debug=debug) if retCode != 0: raise RuntimeError("Error Copying FG table") # Special editing if parms["doEditList"] and not check: mess = "Special editing" printMess(mess, logFile) for edt in parms["editList"]: UV.PFlag(uv,err,timeRange=[dhms2day(edt["timer"][0]),dhms2day(edt["timer"][1])], \ flagVer=parms["editFG"], Ants=edt["Ant"], Chans=edt["Chans"], IFs=edt["IFs"], \ Stokes=edt["Stokes"], Reason=edt["Reason"]) OErr.printErrMsg(err, "Error Flagging") # Quack to remove data from start and end of each scan if parms["doQuack"]: retCode = EVLAQuack (uv, err, begDrop=parms["quackBegDrop"], endDrop=parms["quackEndDrop"], \ Reason=parms["quackReason"], \ logfile=logFile, check=check, debug=debug) if retCode != 0: raise RuntimeError("Error Quacking data") # Flag antennas shadowed by others? if parms["doShad"]: retCode = EVLAShadow (uv, err, shadBl=parms["shadBl"], \ logfile=logFile, check=check, debug=debug) if retCode != 0: raise RuntimeError("Error Shadow flagging data") # Median window time editing, for RFI impulsive in time if parms["doMedn"]: mess = "Median window time editing, for RFI impulsive in time:" printMess(mess, logFile) retCode = EVLAMedianFlag (uv, " ", err, noScrat=noScrat, nThreads=nThreads, \ avgTime=parms["avgTime"], avgFreq=parms["avgFreq"], chAvg= parms["chAvg"], \ timeWind=parms["timeWind"], flagVer=2,flagSig=parms["mednSigma"], \ logfile=logFile, check=check, debug=False) if retCode != 0: raise RuntimeError("Error in MednFlag") # Median window frequency editing, for RFI impulsive in frequency if parms["doFD1"]: mess = "Median window frequency editing, for RFI impulsive in frequency:" printMess(mess, logFile) retCode = EVLAAutoFlag (uv, " ", err, flagVer=2, doCalib=-1, doBand=-1, \ timeAvg=parms["FD1TimeAvg"], \ doFD=True, FDmaxAmp=1.0e20, FDmaxV=1.0e20, FDwidMW=parms["FD1widMW"], \ FDmaxRMS=[1.0e20,0.1], FDmaxRes=parms["FD1maxRes"], \ FDmaxResBL= parms["FD1maxRes"], \ nThreads=nThreads, logfile=logFile, check=check, debug=debug) if retCode != 0: raise RuntimeError("Error in AutoFlag") # RMS/Mean editing for calibrators if parms["doRMSAvg"]: mess = "RMS/Mean editing for calibrators:" printMess(mess, logFile) clist = [] # Calibrator list for s in parms["ACals"]: if s['Source'] not in clist: clist.append(s['Source']) for s in parms["PCals"]: if s['Source'] not in clist: clist.append(s['Source']) for s in parms["DCals"]: if s['Source'] not in clist: clist.append(s['Source']) retCode = EVLAAutoFlag (uv, clist, err, flagVer=2, doCalib=-1, doBand=-1, \ RMSAvg=parms["RMSAvg"], timeAvg=parms["RMSTimeAvg"], \ nThreads=nThreads, logfile=logFile, check=check, debug=debug) if retCode != 0: raise RuntimeError("Error in AutoFlag") # Need to find a reference antenna? See if we have saved it? if (parms["refAnt"] <= 0): refAnt = FetchObject(project + "_" + session + "_" + band + ".refAnt.pickle") if refAnt: parms["refAnt"] = refAnt # Use bandpass calibrator and center half of each spectrum if parms["refAnt"] <= 0: mess = "Find best reference antenna: run Calib on BP Cal(s) " printMess(mess, logFile) parms["refAnt"] = EVLAGetRefAnt(uv, parms["BPCals"], err, flagVer=2, \ solInt=parms["bpsolint1"], nThreads=nThreads, \ logfile=logFile, check=check, debug=debug) if err.isErr: raise RuntimeError("Error finding reference antenna") if parms["refAnts"][0] <= 0: parms["refAnts"][0] = parms["refAnt"] mess = "Picked reference antenna " + str(parms["refAnt"]) printMess(mess, logFile) # Save it ParmsPicklefile = project + "_" + session + "_" + band + ".Parms.pickle" # Where results saved SaveObject(parms, ParmsPicklefile, True) refAntPicklefile = project + "_" + session + "_" + band + ".refAnt.pickle" # Where results saved SaveObject(parms["refAnt"], refAntPicklefile, True) # Plot Raw, edited data? if parms["doRawSpecPlot"] and parms["plotSource"]: mess = "Raw Spectral plot for: " + parms["plotSource"] printMess(mess, logFile) plotFile = "./" + fileRoot + "RawSpec.ps" retCode = EVLASpectrum(uv, parms["plotSource"], parms["plotTime"], plotFile, parms["refAnt"], err, \ Stokes=["RR","LL"], doband=-1, \ check=check, debug=debug, logfile=logFile ) if retCode != 0: raise RuntimeError("Error in Plotting spectrum") EVLAAddOutFile(plotFile, 'project', 'Pipeline log file') # delay calibration if parms["doDelayCal"] and parms["DCals"] and not check: plotFile = "./" + fileRoot + "DelayCal.ps" retCode = EVLADelayCal(uv, parms["DCals"], err, \ BChan=parms["delayBChan"], EChan=parms["delayEChan"], \ doCalib=2, flagVer=2, doBand=-1, \ solInt=parms["delaySolInt"], smoTime=1.0/60.0, \ refAnts=[parms["refAnt"]], doTwo=parms["doTwo"], doZeroPhs=parms["delayZeroPhs"], \ doPlot=parms["doSNPlot"], plotFile=plotFile, \ nThreads=nThreads, noScrat=noScrat, \ logfile=logFile, check=check, debug=debug) if retCode != 0: raise RuntimeError("Error in delay calibration") # Plot corrected data? if parms["doSpecPlot"] and parms["plotSource"]: plotFile = "./" + fileRoot + "DelaySpec.ps" retCode = EVLASpectrum(uv, parms["plotSource"], parms["plotTime"], \ plotFile, parms["refAnt"], err, \ Stokes=["RR","LL"], doband=-1, \ check=check, debug=debug, logfile=logFile ) if retCode != 0: raise RuntimeError("Error in Plotting spectrum") # Bandpass calibration if parms["doBPCal"] and parms["BPCals"]: retCode = EVLABPCal(uv, parms["BPCals"], err, noScrat=noScrat, solInt1=parms["bpsolint1"], \ solInt2=parms["bpsolint2"], solMode=parms["bpsolMode"], \ BChan1=parms["bpBChan1"], EChan1=parms["bpEChan1"], \ BChan2=parms["bpBChan2"], EChan2=parms["bpEChan2"], ChWid2=parms["bpChWid2"], \ doCenter1=parms["bpDoCenter1"], refAnt=parms["refAnt"], \ UVRange=parms["bpUVRange"], doCalib=2, gainUse=0, flagVer=2, doPlot=False, \ nThreads=nThreads, logfile=logFile, check=check, debug=debug) if retCode != 0: raise RuntimeError("Error in Bandpass calibration") # Plot corrected data? if parms["doSpecPlot"] and parms["plotSource"]: plotFile = "./" + fileRoot + "BPSpec.ps" retCode = EVLASpectrum(uv, parms["plotSource"], parms["plotTime"], plotFile, \ parms["refAnt"], err, Stokes=["RR","LL"], doband=1, \ check=check, debug=debug, logfile=logFile ) if retCode != 0: raise RuntimeError("Error in Plotting spectrum") # Constrain X/Y gain if parms["doXYRelGain"] and (parms["XYRelGainCal"] != None): mess = "Calibrate X/Y relative gain:" printMess(mess, logFile) plotFile = "./" + fileRoot + "XYGainCal.ps" retCode = EVLACalAP (uv, [], parms["XYRelGainCal"], err, doCalib=2, doBand=1, BPVer=1, flagVer=2, flagFail=False, \ BChan=parms["ampBChan"], EChan=parms["ampEChan"], \ solInt=parms["solInt"], solSmo=1440., ampScalar=parms["ampScalar"], \ doPlot=parms["doSNPlot"], plotFile=plotFile, refAnt=parms["refAnt"], \ nThreads=nThreads, noScrat=noScrat, logfile=logFile, check=check, debug=debug) if retCode != 0: raise RuntimeError("Error calibrating X/Y gain") parms["avgPol"] = True # Fix X/Y gain # Amp & phase Calibrate if parms["doAmpPhaseCal"]: plotFile = "./" + fileRoot + "APCal.ps" retCode = EVLACalAP (uv, [], parms["ACals"], err, PCals=parms["PCals"], doCalib=2, doBand=1, BPVer=1, flagVer=2, avgPol=parms["avgPol"], \ BChan=parms["ampBChan"], EChan=parms["ampEChan"], \ solInt=parms["solInt"], solSmo=parms["solSmo"], ampScalar=parms["ampScalar"], \ doAmpEdit=parms["doAmpEdit"], ampSigma=parms["ampSigma"], \ ampEditFG=parms["ampEditFG"], \ doPlot=parms["doSNPlot"], plotFile=plotFile, refAnt=parms["refAnt"], \ nThreads=nThreads, noScrat=noScrat, logfile=logFile, check=check, debug=debug) if retCode != 0: raise RuntimeError("Error calibrating") # More editing if parms["doAutoFlag"]: mess = "Post calibration editing:" printMess(mess, logFile) # if going to redo then only calibrators if parms["doRecal"]: # Only calibrators clist = [] for DCal in DCals: if DCal["Source"] not in clist: clist.append(DCal["Source"]) for PCal in PCals: if PCal["Source"] not in clist: clist.append(DCal["Source"]) for ACal in ACals: if ACal["Source"] not in clist: clist.append(ACal["Source"]) else: clist = [] retCode = EVLAAutoFlag (uv, clist, err, flagVer=2, \ doCalib=2, gainUse=0, doBand=1, BPVer=1, \ IClip=parms["IClip"], minAmp=parms["minAmp"], timeAvg=parms["timeAvg"], \ doFD=parms["doAFFD"], FDmaxAmp=parms["FDmaxAmp"], FDmaxV=parms["FDmaxV"], \ FDwidMW=parms["FDwidMW"], FDmaxRMS=parms["FDmaxRMS"], \ FDmaxRes=parms["FDmaxRes"], FDmaxResBL=parms["FDmaxResBL"], \ FDbaseSel=parms["FDbaseSel"], \ nThreads=nThreads, logfile=logFile, check=check, debug=debug) if retCode != 0: raise RuntimeError("Error in AutoFlag") # Redo the calibration using new flagging? if parms["doBPCal2"] == None: parms["doBPCal2"] = parms["doBPCal"] if parms["doDelayCal2"] == None: parms["doDelayCal2"] = parms["doDelayCal2"] if parms["doXYRelGain2"] == None: parms["doXYRelGain2"] = parms["doXYRelGain"] if parms["doAmpPhaseCal2"] == None: parms["doAmpPhaseCal2"] = parms["doAmpPhaseCal"] if parms["doAutoFlag2"] == None: parms["doAutoFlagCal2"] = parms["doAutoFlag"] if parms["doRecal"]: mess = "Redo calibration:" printMess(mess, logFile) EVLAClearCal(uv, err, doGain=True, doFlag=False, doBP=True, check=check, logfile=logFile) OErr.printErrMsg(err, "Error resetting calibration") # Delay recalibration if parms["doDelayCal2"] and parms["DCals"] and not check: plotFile = "./" + fileRoot + "DelayCal2.ps" retCode = EVLADelayCal(uv, parms["DCals"], err, \ BChan=parms["delayBChan"], EChan=parms["delayEChan"], \ doCalib=2, flagVer=2, doBand=-1, \ solInt=parms["delaySolInt"], smoTime=1.0/60.0, \ refAnts=[parms["refAnt"]], doTwo=parms["doTwo"], \ doZeroPhs=parms["delayZeroPhs"], \ doPlot=parms["doSNPlot"], plotFile=plotFile, \ nThreads=nThreads, noScrat=noScrat, \ logfile=logFile, check=check, debug=debug) if retCode != 0: raise RuntimeError("Error in delay calibration") # Plot corrected data? if parms["doSpecPlot"] and parms["plotSource"]: plotFile = "./" + fileRoot + "DelaySpec2.ps" retCode = EVLASpectrum(uv, parms["plotSource"], parms["plotTime"], plotFile, parms["refAnt"], err, \ Stokes=["RR","LL"], doband=-1, \ check=check, debug=debug, logfile=logFile ) if retCode != 0: raise RuntimeError("Error in Plotting spectrum") # Bandpass calibration if parms["doBPCal2"] and parms["BPCals"]: retCode = EVLABPCal(uv, parms["BPCals"], err, noScrat=noScrat, solInt1=parms["bpsolint1"], \ solInt2=parms["bpsolint2"], solMode=parms["bpsolMode"], \ BChan1=parms["bpBChan1"], EChan1=parms["bpEChan1"], \ BChan2=parms["bpBChan2"], EChan2=parms["bpEChan2"], ChWid2=parms["bpChWid2"], \ doCenter1=parms["bpDoCenter1"], refAnt=parms["refAnt"], \ UVRange=parms["bpUVRange"], doCalib=2, gainUse=0, flagVer=2, doPlot=False, \ nThreads=nThreads, logfile=logFile, check=check, debug=debug) if retCode != 0: raise RuntimeError("Error in Bandpass calibration") # Plot corrected data? if parms["doSpecPlot"] and parms["plotSource"]: plotFile = "./" + fileRoot + "BPSpec2.ps" retCode = EVLASpectrum(uv, parms["plotSource"], parms["plotTime"], plotFile, parms["refAnt"], err, \ Stokes=["RR","LL"], doband=1, \ check=check, debug=debug, logfile=logFile ) if retCode != 0: raise RuntimeError("Error in Plotting spectrum") # Constrain X/Y gain parms["avgPol"] = False # Fix X/Y gain? if parms["doXYRelGain2"] and (parms["XYRelGainCal"] != None): mess = "Calibrate X/Y relative gain:" printMess(mess, logFile) plotFile = "./" + fileRoot + "XYGainCal2.ps" retCode = EVLACalAP (uv, [], parms["XYRelGainCal"], err, doCalib=2, doBand=1, BPVer=1, flagVer=2, timeRange=parms["XYRelGainCalTime"], \ BChan=parms["ampBChan"], EChan=parms["ampEChan"], flagFail=False, \ solInt=parms["solInt"], solSmo=1440., ampScalar=parms["ampScalar"], \ doPlot=parms["doSNPlot"], plotFile=plotFile, refAnt=parms["refAnt"], \ nThreads=nThreads, noScrat=noScrat, logfile=logFile, check=check, debug=debug) if retCode != 0: raise RuntimeError("Error calibrating X/Y gain") parms["avgPol"] = True # Fix X/Y gain # Amp & phase Recalibrate if parms["doAmpPhaseCal2"]: plotFile = "./" + fileRoot + "APCal2.ps" retCode = EVLACalAP (uv, [], parms["ACals"], err, PCals=parms["PCals"], \ doCalib=2, doBand=1, BPVer=1, flagVer=2,avgPol=parms["avgPol"], \ BChan=parms["ampBChan"], EChan=parms["ampEChan"], \ solInt=parms["solInt"], solSmo=parms["solSmo"], ampScalar=parms["ampScalar"], \ doAmpEdit=parms["doAmpEdit"], ampSigma=parms["ampSigma"], \ ampEditFG=parms["ampEditFG"], \ doPlot=parms["doSNPlot"], plotFile=plotFile, refAnt=parms["refAnt"], \ noScrat=noScrat, nThreads=nThreads, logfile=logFile, check=check, debug=debug) if retCode != 0: raise RuntimeError("Error calibrating") # More editing if parms["doAutoFlag2"]: mess = "Post recalibration editing:" printMess(mess, logFile) retCode = EVLAAutoFlag (uv, [], err, flagVer=2, \ doCalib=2, gainUse=0, doBand=1, BPVer=1, \ IClip=parms["IClip"], minAmp=parms["minAmp"], timeAvg=parms["timeAvg"], \ doFD=parms["doAFFD"], FDmaxAmp=parms["FDmaxAmp"], FDmaxV=parms["FDmaxV"], \ FDwidMW=parms["FDwidMW"], FDmaxRMS=parms["FDmaxRMS"], \ FDmaxRes=parms["FDmaxRes"], FDmaxResBL= parms["FDmaxResBL"], \ FDbaseSel=parms["FDbaseSel"], \ nThreads=nThreads, logfile=logFile, check=check, debug=debug) if retCode != 0: raise RuntimeError("Error in AutoFlag") # end recal # Calibrate and average data if parms["doCalAvg"]: retCode = EVLACalAvg (uv, avgClass, parms["seq"], parms["CalAvgTime"], err, \ flagVer=2, doCalib=2, gainUse=0, doBand=1, BPVer=1, doPol=False, \ avgFreq=parms["avgFreq"], chAvg=parms["chAvg"], \ BChan=parms["CABChan"], EChan=parms["CAEChan"], \ BIF=parms["CABIF"], EIF=parms["CAEIF"], Compress=parms["Compress"], \ nThreads=nThreads, logfile=logFile, check=check, debug=debug) if retCode != 0: raise RuntimeError("Error in CalAvg") # Get calibrated/averaged data if not check: uv = UV.newPAUV("AIPS UV DATA", EVLAAIPSName(project, session), avgClass[0:6], \ disk, parms["seq"], True, err) if err.isErr: OErr.printErrMsg(err, "Error creating cal/avg AIPS data") # XClip if parms["XClip"] and parms["XClip"] > 0.0: mess = "Cross Pol clipping:" printMess(mess, logFile) retCode = EVLAAutoFlag (uv, [], err, flagVer=-1, flagTab=1, \ doCalib=2, gainUse=0, doBand=-1, maxBad=1.0, \ XClip=parms["XClip"], timeAvg=1./60., \ nThreads=nThreads, logfile=logFile, check=check, debug=debug) if retCode != 0: raise RuntimeError("Error in AutoFlag") # R-L delay calibration cal if needed, if parms["doRLDelay"] and parms["RLDCal"][0][0] != None: if parms["rlrefAnt"] <= 0: parms["rlrefAnt"] = parms["refAnt"] # parms["rlDoBand"] if before average, BPVer=parms["rlBPVer"], retCode = EVLARLDelay(uv, err,\ RLDCal=parms["RLDCal"], BChan=parms["rlBChan"], \ EChan=parms["rlEChan"], UVRange=parms["rlUVRange"], \ soucode=parms["rlCalCode"], doCalib=parms["rlDoCal"], gainUse=parms["rlgainUse"], \ timerange=parms["rltimerange"], \ # NOT HERE doBand=parms["rlDoBand"], BPVer=parms["rlBPVer"], \ flagVer=parms["rlflagVer"], \ refAnt=parms["rlrefAnt"], doPol=False, \ nThreads=nThreads, noScrat=noScrat, logfile=logFile, \ check=check, debug=debug) if retCode != 0: raise RuntimeError("Error in R-L delay calibration") # Polarization calibration if parms["doPolCal"]: if parms["PCRefAnt"] <= 0: parms["PCRefAnt"] = parms["refAnt"] retCode = EVLAPolCal(uv, parms["PCInsCals"], err, \ doCalib=2, gainUse=0, doBand=-1, flagVer=0, \ fixPoln=parms["PCFixPoln"], pmodel=parms["PCpmodel"], avgIF=parms["PCAvgIF"], \ solInt=parms["PCSolInt"], refAnt=parms["PCRefAnt"], solType=parms["PCSolType"], \ ChInc=parms["PCChInc"], ChWid=parms["PCChWid"], \ nThreads=nThreads, check=check, debug=debug, noScrat=noScrat, logfile=logFile) if retCode != 0 and (not check): raise RuntimeError("Error in polarization calibration: " + str(retCode)) # end poln cal. # R-L phase calibration cal., creates new BP table if parms["doRLCal"] and parms["RLDCal"][0][0] != None: plotFile = "./" + fileRoot + "RLSpec2.ps" if parms["rlrefAnt"] <= 0: parms["rlrefAnt"] = parms["refAnt"] retCode = EVLARLCal(uv, err,\ RLDCal=parms["RLDCal"], BChan=parms["rlBChan"], EChan=parms["rlEChan"], UVRange=parms["rlUVRange"], \ ChWid2=parms["rlChWid"], solInt1=parms["rlsolint1"], solInt2=parms["rlsolint2"], \ RLPCal=parms["RLPCal"], RLPhase=parms["RLPhase"], \ RM=parms["RLRM"], CleanRad=parms["rlCleanRad"], \ calcode=parms["rlCalCode"], doCalib=parms["rlDoCal"], gainUse=parms["rlgainUse"], \ timerange=parms["rltimerange"], FOV=parms["rlFOV"], \ doBand=-1, BPVer=1, flagVer=parms["rlflagVer"], \ refAnt=parms["rlrefAnt"], doPol=parms["doPol"], PDVer=parms["PDVer"], \ doPlot=parms["doSpecPlot"], plotFile=plotFile, \ nThreads=nThreads, noScrat=noScrat, logfile=logFile, \ check=check, debug=debug) if retCode != 0: raise RuntimeError("Error in RL phase spectrum calibration") # VClip if parms["VClip"] and parms["VClip"] > 0.0: mess = "VPol clipping:" printMess(mess, logFile) retCode = EVLAAutoFlag (uv, [], err, flagVer=-1, flagTab=1, \ doCalib=2, gainUse=0, doBand=-1, \ VClip=parms["VClip"], timeAvg=parms["timeAvg"], \ nThreads=nThreads, logfile=logFile, check=check, debug=debug) if retCode != 0: raise RuntimeError("Error in AutoFlag VClip") # Plot corrected data? if parms["doSpecPlot"] and parms["plotSource"]: plotFile = "./" + fileRoot + "Spec.ps" retCode = EVLASpectrum(uv, parms["plotSource"], parms["plotTime"], \ plotFile, parms["refAnt"], err, \ Stokes=["RR","LL"], doband=-1, \ check=check, debug=debug, logfile=logFile ) if retCode != 0: raise RuntimeError("Error in Plotting spectrum") # Image targets if parms["doImage"]: # If targets not specified, image all if len(parms["targets"]) <= 0: slist = EVLAAllSource(uv, err, logfile=logFile, check=check, debug=debug) else: slist = parms["targets"] EVLAImageTargets (uv, err, Sources=slist, seq=parms["seq"], sclass=outIClass, \ doCalib=2, doBand=1, flagVer=1, doPol=parms["doPol"], PDVer=parms["PDVer"], \ Stokes=parms["Stokes"], FOV=parms["FOV"], Robust=parms["Robust"], Niter=parms["Niter"], \ CleanRad=parms["CleanRad"], minFlux=parms["minFlux"], \ maxPSCLoop=parms["maxPSCLoop"], minFluxPSC=parms["minFluxPSC"], \ solPInt=parms["solPInt"], solPMode=parms["solPMode"], solPType=parms["solPType"], \ maxASCLoop=parms["maxASCLoop"], minFluxASC=parms["minFluxASC"], \ solAInt=parms["solAInt"], solAMode=parms["solAMode"], solAType=parms["solAType"], \ avgPol=parms["avgPol"], avgIF=parms["avgIF"], minSNR = 4.0, refAnt=parms["refAnt"], \ do3D=parms["do3D"], BLFact=parms["BLFact"], BLchAvg=parms["BLchAvg"], \ doMB=parms["doMB"], norder=parms["MBnorder"], maxFBW=parms["MBmaxFBW"], \ PBCor=parms["PBCor"],antSize=parms["antSize"], Beam=parms["Beam"], \ nTaper=parms["nTaper"], Tapers=parms["Tapers"], \ nThreads=nThreads, noScrat=noScrat, logfile=logFile, check=check, debug=debug) # End image # Get report on sources if parms["doReport"]: # If targets not specified, do all if len(parms["targets"]) <= 0: slist = EVLAAllSource(uv, err, logfile=logFile, check=check, debug=debug) else: slist = parms["targets"] Report = EVLAReportTargets(uv, err, Sources=slist, seq=parms["seq"], sclass=outIClass, \ Stokes=parms["Stokes"], logfile=logFile, check=check, debug=debug) # Save to pickle jar ReportPicklefile = "./" + fileRoot + "Report.pickle" # Where results saved SaveObject(Report, ReportPicklefile, True) # Write results, cleanup # Save cal/average UV data? if parms["doSaveUV"] and (not check): Aname = EVLAAIPSName(project, session) cno = AIPSDir.PTestCNO(disk, user, Aname, avgClass[0:6], "UV", parms["seq"], err) if cno > 0: uvt = UV.newPAUV("AIPS CAL UV DATA", Aname, avgClass, disk, parms["seq"], True, err) filename = parms["project"] + parms["session"] + parms[ "band"] + "Cal.uvtab" fuv = EVLAUVFITS(uvt, filename, 0, err, compress=parms["Compress"], logfile=logFile) EVLAAddOutFile(filename, 'project', "Calibrated Averaged UV data") # Save list of output files EVLASaveOutFiles() del uvt # Save raw UV data tables? if parms["doSaveTab"] and (not check): Aname = EVLAAIPSName(project, session) cno = AIPSDir.PTestCNO(disk, user, Aname, dataClass[0:6], "UV", parms["seq"], err) if cno > 0: uvt = UV.newPAUV("AIPS RAW UV DATA", Aname, dataClass[0:6], disk, parms["seq"], True, err) filename = parms["project"] + parms["session"] + parms[ "band"] + "CalTab.uvtab" fuv = EVLAUVFITSTab(uvt, filename, 0, err, logfile=logFile) EVLAAddOutFile(filename, 'project', "Calibrated AIPS tables") del uvt # Write History filename = project + '_' + session + '_' + band + ".History.text" OTObit.PrintHistory(uv, file=filename) EVLAAddOutFile(filename, 'project', "Processing history of calibrated data") # Save list of output files EVLASaveOutFiles() # Imaging results # If targets not specified, save all if len(parms["targets"]) <= 0: slist = EVLAAllSource(uv, err, logfile=logFile, check=check, debug=debug) else: slist = parms["targets"] for target in slist: if parms["doSaveImg"] and (not check): for s in parms["Stokes"]: oclass = s + outIClass[1:] outname = target # Test if image exists cno = AIPSDir.PTestCNO(disk, user, outname, oclass, "MA", parms["seq"], err) if cno <= 0: continue x = Image.newPAImage("out", outname, oclass, disk, parms["seq"], True, err) outfile = "./" + fileRoot + target + "." + oclass + ".fits" xf = EVLAImFITS(x, outfile, 0, err, logfile=logFile) EVLAAddOutFile(outfile, target, 'Image of ' + target) # Statistics zz = imstat(x, err, logfile=logFile) # end writing loop # Save list of output files EVLASaveOutFiles() OErr.printErrMsg(err, "Writing output") # Contour plots if parms["doKntrPlots"]: mess = "INFO --> Contour plots (doKntrPlots)" printMess(mess, logFile) EVLAKntrPlots(err, imName=parms["targets"], project=project, session=session, band=band, disk=disk, debug=debug) # Save list of output files EVLASaveOutFiles() elif debug: mess = "Not creating contour plots ( doKntrPlots = " + str( parms["doKntrPlots"]) + " )" printMess(mess, logFile) # Source uv plane diagnostic plots if parms["doDiagPlots"]: mess = "INFO --> Diagnostic plots (doDiagPlots)" printMess(mess, logFile) # Get the highest number avgClass catalog file Aname = EVLAAIPSName(project, session) uvc = None if not check: uvname = project + "_" + session + "_" + band + "_Cal" uvc = UV.newPAUV(uvname, Aname, avgClass, disk, parms["seq"], True, err) EVLADiagPlots( uvc, err, cleanUp=parms["doCleanup"], \ project=project, session=session, band=band, \ logfile=logFile, check=check, debug=debug ) # Save list of output files EVLASaveOutFiles() elif debug: mess = "Not creating diagnostic plots ( doDiagPlots = " + str( parms["doDiagPlots"]) + " )" printMess(mess, logFile) # Save metadata srcMetadata = None projMetadata = None if parms["doMetadata"]: mess = "INFO --> Save metadata (doMetadata)" printMess(mess, logFile) uvc = None if not uvc: # Get calibrated/averaged data Aname = EVLAAIPSName(project, session) uvname = project + "_" + session + "_" + band + "_Cal" uvc = UV.newPAUV(uvname, Aname, avgClass, disk, parms["seq"], True, err) if err.isErr: OErr.printErrMsg(err, "Error creating cal/avg AIPS data") # Get source metadata; save to pickle file srcMetadata = EVLASrcMetadata( uvc, err, Sources=parms["targets"], seq=parms["seq"], \ sclass=outIClass, Stokes=parms["Stokes"],\ logfile=logFile, check=check, debug=debug ) picklefile = "./" + fileRoot + ".SrcReport.pickle" SaveObject(srcMetadata, picklefile, True) EVLAAddOutFile(picklefile, 'project', 'All source metadata') # Get project metadata; save to pickle file projMetadata = EVLAProjMetadata( uvc, AIPS_VERSION, err, \ PCals=parms["PCals"], ACals=parms["ACals"], \ BPCals=parms["BPCals"], DCals=parms["DCals"], \ project = project, session = session, band = band, \ dataInUVF = parms["archRoot"], archFileID = 66666 ) picklefile = "./" + fileRoot + ".ProjReport.pickle" SaveObject(projMetadata, picklefile, True) EVLAAddOutFile(picklefile, 'project', 'Project metadata') else: # Fetch from pickle jar picklefile = "./" + fileRoot + ".SrcReport.pickle" srcMetadata = FetchObject(picklefile) picklefile = "./" + fileRoot + ".ProjReport.pickle" projMetadata = FetchObject(picklefile) # Write report if parms["doHTML"]: mess = "INFO --> Write HTML report (doHTML)" printMess(mess, logFile) EVLAHTMLReport( projMetadata, srcMetadata, \ outfile="./"+fileRoot+".report.html", \ logFile=logFile ) # Write VOTable if parms["doVOTable"]: mess = "INFO --> Write VOTable (doVOTable)" printMess(mess, logFile) EVLAAddOutFile('VOTable.xml', 'project', 'VOTable report') EVLAWriteVOTable(projMetadata, srcMetadata, filename='VOTable.xml') # Save list of output files EVLASaveOutFiles() # Cleanup - delete AIPS files if parms["doCleanup"] and (not check): mess = "INFO --> Clean up (doCleanup)" printMess(mess, logFile) # Delete target images # How many Stokes images nstok = len(parms["Stokes"]) for istok in range(0, nstok): oclass = parms["Stokes"][istok:istok + 1] + outIClass[1:] AllDest(err, disk=disk, Aseq=parms["seq"], Aclass=oclass) # Delete initial UV data Aname = EVLAAIPSName(project, session) # Test if data exists cno = AIPSDir.PTestCNO(disk, user, Aname, dataClass[0:6], "UV", parms["seq"], err) if cno > 0: uvt = UV.newPAUV("AIPS RAW UV DATA", Aname, dataClass[0:6], disk, parms["seq"], True, err) uvt.Zap(err) del uvt if err.isErr: OErr.printErrMsg(err, "Error deleting raw AIPS data") # Zap calibrated/averaged data # Test if data exists cno = AIPSDir.PTestCNO(disk, user, Aname, avgClass[0:6], "UV", parms["seq"], err) if cno > 0: uvt = UV.newPAUV("AIPS CAL UV DATA", Aname, avgClass[0:6], disk, parms["seq"], True, err) uvt.Zap(err) del uvt if err.isErr: OErr.printErrMsg(err, "Error deleting cal/avg AIPS data") # Zap UnHanned data if present loadClass = parms["band"] + "Raw" # Test if image exists cno = AIPSDir.PTestCNO(disk, user, Aname, loadClass[0:6], "UV", parms["seq"], err) if cno > 0: uvt = UV.newPAUV("AIPS CAL UV DATA", Aname, loadClass[0:6], disk, parms["seq"], True, err) uvt.Zap(err) del uvt if err.isErr: OErr.printErrMsg(err, "Error deleting cal/avg AIPS data") OErr.printErrMsg(err, "Writing output/cleanup") # Shutdown mess = "Finished project "+parms["project"]+" session "+parms["session"]+ \ " "+parms["band"]+" Band"+" AIPS user no. "+str(AIPS.userno) printMess(mess, logFile) OErr.printErr(err) OSystem.Shutdown(ObitSys)
doBP=doBP, check=check, logfile=logFile) OErr.printErrMsg(err, "Error resetting calibration") # Copy FG 1 to FG 2 if doCopyFG: retCode = VLBACopyFG(uv, err, logfile=logFile, check=check, debug=debug) if retCode != 0: raise RuntimeError, "Error Copying FG table" # Special editing if doEditList and not check: for edt in editList: UV.PFlag(uv,err,timeRange=[dhms2day(edt["timer"][0]),dhms2day(edt["timer"][1])], \ flagVer=editFG, Ants=edt["Ant"], Chans=edt["Chans"], IFs=edt["IFs"], \ Stokes=edt["Stokes"], Reason=edt["Reason"]) OErr.printErrMsg(err, "Error Flagging") # Quack to remove data from start and end of each scan if doQuack: retCode = VLBAQuack (uv, err, begDrop=quackBegDrop, endDrop=quackEndDrop, Reason=quackReason, \ logfile=logFile, check=check, debug=debug) if retCode != 0: raise RuntimeError, "Error Quacking data" # Add velocity/rest freq info if doFreqInfo: retCode = VLBAFreqInfo(uv, restfreq, srcVel,
def pipeline(aipsSetup, parmFile): """ VLBA Continuum pipeline. * *aipsSetup* = AIPS setup file * *parmFile* = pipeline input parameters file """ ############################# Initialize OBIT ########################################## noScrat = [] exec(open(aipsSetup).read()) VLBAAddOutFile(aipsSetup, 'project', "Obit's AIPS setup file") ############################# Default parameters ########################################## # Define data project = "Unspecified" # Project name (12 char or less, used as AIPS Name) session = "?" # Project session code band = "?" # Observing band logFile = project + "_" + session + "_" + band + ".log" # Processing log file seq = 1 # AIPS sequence number gain = 0.10 # CLEAN loop gain doLoadIDI = True # Load data from IDI FITS? doLoadUVF = False # Load the "AIPS Friendly" (uvfits) FITS version dataInUVF = None # Input uvfits data file name dataInIDI = None # Input FITS-IDI file or list dataClass = "Raw" # AIPS class of raw uv data Compress = False # Use compressed UV data? calInt = 0.15 # Calibration table interval in min. wtThresh = 0.8 # Data weight threshold check = False # Only check script, don't execute tasks debug = False # run tasks debug prtLv = 2 # Print level # Initialize parameters parms = VLBAInitContParms() ############################# Set Project Processing parameters ################## exec(open(parmFile).read()) VLBAAddOutFile(parmFile, 'project', 'Pipeline input parameters') ################################## Process ##################################### # Init cal pickle jars goodCalPicklefile = project + "_" + session + "_" + band + "_GoodCal.pickle" # Where results saved # Default "best" calibration goodCal = { "Source": " ", "souID": 0, "timeRange": (0.0, 100.0), "Fract": 0.0, "SNR": 0.0, "bestRef": 0 } # Save initial default if it doesn't already exist SaveObject(goodCal, goodCalPicklefile, False) VLBAAddOutFile(goodCalPicklefile, 'project', 'Best calibrator information') OKCalPicklefile = project + "_" + session + "_" + band + "_OKCal.pickle" # Where results saved SaveObject(parms["contCals"], OKCalPicklefile, False) VLBAAddOutFile(OKCalPicklefile, 'project', 'List of calibrators') targetsPicklefile = project + "_" + session + "_" + band + "_targets.pickle" # Where results saved if (not parms["targets"]) and os.path.exists(targetsPicklefile): parms["targets"] = FetchObject(targetsPicklefile) else: SaveObject(parms["targets"], targetsPicklefile, False) VLBAAddOutFile(targetsPicklefile, 'project', 'List of targets') # Load the outputs pickle jar VLBAFetchOutFiles() # Logging directly to logFile OErr.PInit(err, prtLv, logFile) retCode = 0 VLBAAddOutFile(logFile, 'project', 'Pipeline log file') mess = "Start project " + project + " session " + session + " " + band + " Band" + " AIPS user no. " + str( AIPS.userno) printMess(mess, logFile) logger.info(mess) if debug: pydoc.ttypager = pydoc.plainpager # don't page task input displays mess = "Using Debug mode " printMess(mess, logFile) if check: mess = "Only checking script" printMess(mess, logFile) # Load Data from FITS uv = None # Raw data uvc = None # Cal/averaged data if doLoadIDI: logger.info("--> Load IDI data file (doLoadIDI)") if type(dataInIDI) == list: logger.info( "--> Loading a list of IDI files (dataInIDI is a list)\n" + " This data appears to be from the old correlator.\n" + " IDI corrections will be applied when loading is\n" + " complete.") # Read list for dataIn in dataInIDI: uv = VLBAIDILoad(dataIn, project, session, band, dataClass, disk, seq, err, logfile=logFile, \ wtThresh=wtThresh, calInt=calInt, Compress=Compress, \ check=check, debug=debug) if not UV.PIsA(uv): raise RuntimeError("Cannot load " + dataIn) # Fix IDI files: uv = IDIFix.IDIFix(uv, err) seq = uv.Aseq else: logger.info( "--> Loading a single IDI file (dataInIDI is not a list)\n" + " This data appears to be from the DiFX correlator.\n" + " No IDI corrections will be applied.") # Single IDI file uv = VLBAIDILoad(dataInIDI, project, session, band, dataClass, disk, seq, err, logfile=logFile, \ wtThresh=wtThresh, calInt=calInt, Compress=Compress, \ check=check, debug=debug) if not UV.PIsA(uv): raise RuntimeError("Cannot load " + dataInIDI) if doLoadUVF: logger.info("--> Load UVFITS data file (doLoadUVF)") uv = VLBAIDILoad(dataInUVF, project, session, band, dataClass, disk, seq, err, logfile=logFile, \ wtThresh=wtThresh, calInt=calInt, Compress=Compress, \ check=check, debug=debug) # Adding check condition to avoid error when checking if not UV.PIsA(uv) and not check: raise RuntimeError("Cannot load " + dataInUVF) # Otherwise set uv if uv == None and not check: Aname = VLBAAIPSName(project, session) uvname = project + "_" + session + "_" + band uv = UV.newPAUV(uvname, Aname, dataClass, disk, seq, True, err) if err.isErr: OErr.printErrMsg(err, "Error creating AIPS data") # frequency dependent default parameters VLBAInitContFQParms(uv, parms, err, \ logfile=logFile, check=check, debug=debug) # Setup target source list if os.path.exists(targetsPicklefile): parms["targets"] = \ setupSourceList( parms["targets"], uv, err, logFile, check, debug ) logger.debug("parms['targets'] = " + str(parms["targets"])) # Clear any old calibration/editing if parms["doClearTab"]: logger.info("--> Clear old calibration/editing (doClearTab)") VLBAClearCal(uv, err, doGain=parms["doGain"], doFlag=parms["doFlag"], doBP=parms["doBP"], \ check=check, logfile=logFile) OErr.printErrMsg(err, "Error resetting calibration") # Copy FG 1 to FG 2 if parms["doCopyFG"]: logger.info("--> Copy flag (FG) table 1 to flag table 2 (doCopyFG)") retCode = VLBACopyFG(uv, err, logfile=logFile, check=check, debug=debug) if retCode != 0: raise RuntimeError("Error Copying FG table") # Special editing if parms["doEditList"] and not check: logger.info("--> Special editing (doEditList)") for edt in parms["editList"]: UV.PFlag(uv,err,timeRange=[dhms2day(edt["timer"][0]),dhms2day(edt["timer"][1])], \ flagVer=editFG, Ants=edt["Ant"], Chans=edt["Chans"], IFs=edt["IFs"], \ Stokes=edt["Stokes"], Reason=edt["Reason"]) OErr.printErrMsg(err, "Error Flagging") # Quack to remove data from start and end of each scan if parms["doQuack"]: logger.info("--> Remove data from start and end of scans (doQuack)") retCode = VLBAQuack (uv, err, \ begDrop=parms["quackBegDrop"], endDrop=parms["quackEndDrop"], \ Reason=parms["quackReason"], \ logfile=logFile, check=check, debug=debug) if retCode != 0: raise RuntimeError("Error Quacking data") # Median window time editing, for RFI impulsive in time or dropouts if parms["doMedn"]: logger.info("--> Median window time editing (doMedn)") retCode = VLBAMedianFlag (uv, " ", err, noScrat=noScrat, nThreads=nThreads, \ avgTime=parms["mednAvgTime"], avgFreq=parms["mednAvgFreq"], \ chAvg=parms["mednChAvg"], timeWind=parms["mednTimeWind"], \ flagSig=parms["mednSigma"], flagVer=2, \ logfile=logFile, check=check, debug=False) if retCode != 0: raise RuntimeError("Error in MednFlag") # Quantization correction? if parms["doQuantCor"]: logger.info("--> Quantization correction (doQuantCor)") plotFile = project + "_" + session + "_" + band + ".Quant.ps" retCode = VLBAQuantCor(uv, parms["QuantSmo"], parms["QuantFlag"], err, \ doSNPlot=parms["doSNPlot"], plotFile=plotFile, \ logfile=logFile, check=check, debug=debug) if retCode != 0: raise RuntimeError("Error in quantization correcting/flagging") # Parallactic angle correction? if parms["doPACor"]: logger.info("--> Parallactic angle correction (doPACor)") retCode = VLBAPACor(uv, err, noScrat=noScrat, \ logfile=logFile, check=check, debug=debug) if retCode != 0: raise RuntimeError("Error in quantization correcting/flagging") # Opacity/Tsys/gain correction if parms["doOpacCor"]: logger.info("--> Opacity/Tsys/Gain correction (doOpacCor)") plotFile = project + "_" + session + "_" + band + ".Opacity.ps" retCode = VLBAOpacCor(uv, parms["OpacSmoo"], err, \ doSNPlot=parms["doSNPlot"], plotFile=plotFile, \ logfile=logFile, check=check, debug=debug) if retCode != 0: raise RuntimeError("Error in opacity/gain.Tsys correction") VLBASaveOutFiles() # Save plot file in Outfiles # Need to determine a list of calibrators? if (parms["contCals"] == None) or (len(parms["contCals"]) <= 0): logger.info("--> Get list of calibrators (contCals = None or [])") if parms["doFindOK"]: slist = VLBAAllSource(uv, err, logfile=logFile, check=check, debug=debug) parms["contCals"] = VLBAOKCal(uv, parms["minOKFract"], err, \ solInt=parms["findSolInt"], \ calSou=slist, minSNR=parms["minOKSNR"], \ doCalib=-1, flagVer=2, refAnts=parms["refAnts"], \ noScrat=noScrat, nThreads=nThreads, \ logfile=logFile, check=check, debug=debug) if not parms["contCals"] and not check: raise RuntimeError("Error in finding acceptable calibrators") logger.info("Calibrators = " + str(parms["contCals"])) else: # Snatch from pickle jar parms["contCals"] = FetchObject(OKCalPicklefile) # Save contCals to a pickle jar SaveObject(parms["contCals"], OKCalPicklefile, True) # Find best calibration source if parms["doFindCal"]: logger.info("--> Find best calibration source (doFindCal)") goodCal = VLBAGoodCal(uv, err, \ solInt=parms["findSolInt"], timeInt=parms["findTimeInt"], \ calSou=parms["contCals"], \ #CalModel=parms["contCalModel"], \ doCalib=-1, flagVer=2, refAnts=parms["refAnts"], \ noScrat=noScrat, nThreads=nThreads, \ logfile=logFile, check=check, debug=debug) if not goodCal and not check: raise RuntimeError("Error in finding best calibration data") # Save it to a pickle jar SaveObject(goodCal, goodCalPicklefile, True) else: # Fetch from pickle goodCal = FetchObject(goodCalPicklefile) # Apply Phase cals from PC table? if parms["doPCcor"] and not check: logger.info("--> Apply phase cals (doPCcor)") plotFile = project + "_" + session + "_" + band + ".PC.ps" retCode = VLBAPCcor(uv, err, calSou=goodCal["Source"], \ timeRange=goodCal["timeRange"], \ doCalib=-1, flagVer=2, solInt=parms["manPCsolInt"], \ PCin=1, SNout=0, refAnt=goodCal["bestRef"], \ doPCPlot=parms["doPCPlot"], plotFile=plotFile, \ noScrat=noScrat, logfile=logFile, check=check, debug=debug) if retCode != 0: raise RuntimeError("Error in PC calibration") VLBASaveOutFiles() # Save plot file in Outfiles # manual phase cal if parms["doManPCal"] and not check: logger.info("--> Manual phase cal (doManPCal)") plotFile = project + session + band + ".ManPCal.ps" retCode = VLBAManPCal(uv, err, calSou=goodCal["Source"], \ #CalModel=parms["contCalModel"], \ timeRange=goodCal["timeRange"], \ solInt=parms["manPCsolInt"], smoTime=parms["manPCSmoo"], \ refAnts=[goodCal["bestRef"]], doCalib=2, flagVer=2, \ doManPCalPlot=parms["doManPCalPlot"], \ plotFile=plotFile, noScrat=noScrat, \ nThreads=nThreads, logfile=logFile, check=check, debug=debug) if retCode != 0: raise RuntimeError("Error in manual phase calibration") # image cals if parms["doImgCal"] and not check: logger.info("--> Image calibrators (doImgCal)") VLBAImageCals(uv, err, Sources=parms["contCals"], seq=seq, sclass=parms["outCclass"], doCalib=2, flagVer=2, doBand=0, FOV=parms["FOV"], Robust=parms["Robust"], maxPSCLoop=parms["maxPSCLoop"], minFluxPSC=parms["minFluxPSC"], solPInt=parms["solPInt"], solMode=parms["solMode"], maxASCLoop=parms["maxASCLoop"], minFluxASC=parms["minFluxASC"], solAInt=parms["solAInt"], avgPol=parms["avgPol"], avgIF=parms["avgIF"], minSNR=parms["minSNR"], refAnt=goodCal["bestRef"], nThreads=nThreads, noScrat=noScrat, logfile=logFile, check=check, debug=debug) # Rewrite OKCal pickle file because calibrators may have been updated SaveObject(parms["contCals"], OKCalPicklefile, True) if len(parms["contCals"]) <= 0: logger.error( "No calibration sources have been detected! Stopping pipeline." ) raise RuntimeError("No calibration sources have been detected!") # Check if calibrator models now available parms["contCalModel"] = VLBAImageModel(parms["contCals"], parms["outCclass"], disk, seq, err) # delay calibration if parms["doDelayCal"] and not check: logger.info("--> Delay calibration (doDelayCal)") plotFile = project + "_" + session + "_" + band + ".DelayCal.ps" retCode = VLBADelayCal(uv, err, calSou=parms["contCals"], CalModel=parms["contCalModel"], \ doCalib=2, flagVer=2, doBand=0, \ solInt=parms["manPCsolInt"], smoTime=parms["delaySmoo"], \ refAnts=[goodCal["bestRef"]], \ doSNPlot=parms["doSNPlot"], plotFile=plotFile, \ nThreads=nThreads, noScrat=noScrat, logfile=logFile, check=check, debug=debug) if retCode != 0: raise RuntimeError("Error in delay calibration") VLBASaveOutFiles() # Save plot file in Outfiles # Bandpass calibration if needed if parms["doBPCal"] and not check: logger.info("--> Bandpass calibration (doBPCal)") retCode = VLBABPass(uv, goodCal["Source"], err, CalModel=parms["contCalModel"], \ timeRange=goodCal["timeRange"], doCalib=2, flagVer=2, \ noScrat=noScrat, solInt1=parms["bpsolint1"], \ solInt2=parms["bpsolint2"], solMode=parms["bpsolMode"], \ BChan1=parms["bpBChan1"], EChan1=parms["bpEChan1"], BChan2=parms["bpBChan2"], \ EChan2=parms["bpEChan2"], ChWid2=parms["bpChWid2"], \ doCenter1=parms["bpDoCenter1"], refAnt=goodCal["bestRef"], specIndex=parms["specIndex"], \ doAuto = parms["bpdoAuto"], \ nThreads=nThreads, logfile=logFile, check=check, debug=debug) if retCode != 0: raise RuntimeError("Error in Bandpass calibration") # Plot amplitude and phase vs. frequency if parms["doSpecPlot"]: logger.info("--> Spectral plotting (doSpecPlot)") plotFile = project + '_' + session + '_' + band + ".spec.ps" VLBASpecPlot(uv, goodCal, err, doband=1, check=check, plotFile=plotFile, logfile=logFile, debug=debug) VLBASaveOutFiles() # Save plot file in Outfiles # Phase calibration using calibrator models if parms["doPhaseCal"]: logger.info( "--> Phase calibration using calibrator models (doPhaseCal)") plotFile = project + "_" + session + "_" + band + ".PhaseCal0.ps" retCode = VLBAPhaseCal(uv, err, calSou=parms["contCals"], CalModel=parms["contCalModel"], \ doCalib=-1, flagVer=0, doBand=-1, \ refAnt=goodCal["bestRef"], solInt=parms["solPInt"], \ doSNPlot=parms["doSNPlot"], plotFile=plotFile, \ nThreads=nThreads, noScrat=noScrat, logfile=logFile, check=check, debug=debug) if retCode != 0: raise RuntimeError("Error in phase calibration") VLBASaveOutFiles() # Save plot file in Outfiles # Rewrite contCals pickle file because contCals may have been updated SaveObject(parms["contCals"], OKCalPicklefile, True) if len(parms["contCals"]) <= 0: logger.error( "No calibrator sources have been detected! Stopping pipeline.") raise RuntimeError("No calibrator sources have been detected!") # Amplitude calibration if parms["doAmpCal"] and not check: logger.info("--> Amplitude calibration (doAmpCal)") plotFile = project + "_" + session + "_" + band + ".AmpCal.ps" retCode = VLBAAmpCal(uv, err, calSou=parms["contCals"], CalModel=parms["contCalModel"], doCalib=2, flagVer=2, doBand=1, minSumCC=parms["minFluxASC"], refAnt=goodCal["bestRef"], solInt=parms["solAInt"], smoTimeA=2.0, smoTimeP=1. / 60., doSNPlot=parms["doSNPlot"], plotFile=plotFile, nThreads=nThreads, noScrat=noScrat, logfile=logFile, check=check, debug=debug) if retCode != 0: raise RuntimeError("Error in amplitude calibration") VLBASaveOutFiles() # Save plot file in Outfiles # Rewrite contCals pickle file because contCals may have been updated SaveObject(parms["contCals"], OKCalPicklefile, True) if len(parms["contCals"]) <= 0: logger.error( "No calibrator sources have been detected! Stopping pipeline.") raise RuntimeError("No calibrator sources have been detected!") # Calibrate and average data if parms["doCalAvg"]: logger.info("--> Calibration and average data (doCalAvg)") retCode = VLBACalAvg (uv, parms["avgClass"], seq, parms["CalAvgTime"], err, \ flagVer=2, doCalib=2, gainUse=0, doBand=1, BPVer=1, \ BIF=parms["CABIF"], EIF=parms["CAEIF"], \ BChan=parms["CABChan"], EChan=parms["CAEChan"], \ chAvg=parms["chAvg"], avgFreq=parms["avgFreq"], Compress=Compress, \ logfile=logFile, check=check, debug=debug) if retCode != 0: raise RuntimeError("Error in CalAvg") # image targets phase only self-cal - NOTE: actually A&P if parms["doImgTarget"] and not check: logger.info("--> Image targets (doImgTargets)") if not uvc: # Get calibrated/averaged data Aname = VLBAAIPSName(project, session) uvname = project + "_" + session + "_" + band + "_Cal" uvc = UV.newPAUV(uvname, Aname, parms["avgClass"], disk, seq, True, err) if err.isErr: OErr.printErrMsg(err, "Error creating cal/avg AIPS data") logger.debug("parms['targets'] = " + str(parms["targets"])) VLBAImageCals(uv, err, Sources=parms["targets"], seq=seq, sclass=parms["outTclass"], doCalib=2, flagVer=2, doBand=1, FOV=parms["FOV"], Robust=parms["Robust"], maxPSCLoop=parms["maxPSCLoop"], minFluxPSC=parms["minFluxPSC"], solPInt=parms["solPInt"], solMode=parms["solMode"], maxASCLoop=parms["maxASCLoop"], minFluxASC=parms["minFluxASC"], solAInt=parms["solAInt"], avgPol=parms["avgPol"], avgIF=parms["avgIF"], minSNR=parms["minSNR"], refAnt=goodCal["bestRef"], nThreads=nThreads, noScrat=noScrat, logfile=logFile, check=check, debug=debug) # Rewrite targets pickle file because targets may have been updated SaveObject(parms["targets"], targetsPicklefile, True) if len(parms["targets"]) <= 0: logger.error( "No target sources have been detected! Stopping pipeline.") raise RuntimeError("No target sources have been detected!") # Phase calibration using target models if parms["doPhaseCal"]: logger.info("--> Phase calibration using target models (doPhaseCal)") if not uvc: # Get calibrated/averaged data Aname = VLBAAIPSName(project, session) uvname = project + "_" + session + "_" + band + "_Cal" uvc = UV.newPAUV(uvname, Aname, parms["avgClass"], disk, seq, True, err) if err.isErr: OErr.printErrMsg(err, "Error creating cal/avg AIPS data") parms["targetModel"] = VLBAImageModel(parms["targets"], parms["outTclass"], disk, seq, err) plotFile = project + "_" + session + "_" + band + ".PhaseCal.ps" retCode = VLBAPhaseCal(uvc, err, calSou=parms["targets"], CalModel=parms["targetModel"], \ doCalib=-1, flagVer=0, doBand=-1, \ refAnt=goodCal["bestRef"], solInt=parms["manPCsolInt"], \ doSNPlot=parms["doSNPlot"], plotFile=plotFile, \ nThreads=nThreads, noScrat=noScrat, logfile=logFile, check=check, debug=debug) if retCode != 0: raise RuntimeError("Error in phase calibration") VLBASaveOutFiles() # Save plot file in Outfiles # Rewrite targets pickle file because targets may have been updated SaveObject(parms["targets"], targetsPicklefile, True) if len(parms["targets"]) <= 0: logger.error( "No target sources have been detected! Stopping pipeline.") raise RuntimeError("No target sources have been detected!") # Instrumental polarization calibration if parms["doInstPol"]: logger.info("--> Instrumental polarization calibration (doInstPol)") # calibrators defaults to strong calibrator list if not parms["instPolCal"]: instPolCal = contCals else: instPolCal = parms["instPolCal"] if not uvc: # Get calibrated/averaged data Aname = VLBAAIPSName(project, session) uvname = project + "_" + session + "_" + band + "_Cal" uvc = UV.newPAUV(uvname, Aname, parms["avgClass"], disk, seq, True, err) if err.isErr: OErr.printErrMsg(err, "Error creating cal/avg AIPS data") retCode = VLBAPolCal(uvc, instPolCal, err, \ doCalib=2, flagVer=0, doBand=-1, doSetJy=True, \ refAnt=goodCal["bestRef"], solInt=2.0, \ noScrat=noScrat, logfile=logFile, check=check, debug=debug) if retCode != 0: raise RuntimeError("Error in instrumental poln calibration") # RL Phase (EVPA) calibration as BP table if parms["doRLCal"] and parms["RLCal"]: logger.info("--> RL phase calibration (doRLCal)") if not uvc: # Get calibrated/averaged data Aname = VLBAAIPSName(project, session) uvname = project + "_" + session + "_" + band + "_Cal" uvc = UV.newPAUV(uvname, Aname, parms["avgClass"], disk, seq, True, err) if err.isErr: OErr.printErrMsg(err, "Error creating cal/avg AIPS data") retCode = VLBARLCal2(uvc, err, RLPCal=parms["RLCal"], \ doCalib=2, gainUse=2, flagVer=0, doBand=-1, doPol=True, \ refAnt=goodCal["bestRef"], niter=300, FOV=0.02/3600.0, \ nThreads=nThreads, noScrat=noScrat, logfile=logFile, check=check, debug=debug) if retCode != 0: raise RuntimeError("Error in RL phase calibration") # image targets possible with Stokes I(QU) if parms["doImgFullTarget"]: logger.info("--> Image targets (doImgFullTargets)") if not uvc: # Get calibrated/averaged data Aname = VLBAAIPSName(project, session) uvname = project + "_" + session + "_" + band + "_Cal" uvc = UV.newPAUV(uvname, Aname, parms["avgClass"], disk, seq, True, err) if err.isErr: OErr.printErrMsg(err, "Error creating cal/avg AIPS data") VLBAImageTargets(uvc, err, Sources=parms["targets"], seq=seq, sclass=parms["outIclass"], doCalib=2, flagVer=0, doBand=-1, Stokes=parms["Stokes"], FOV=parms["FOV"], Robust=parms["Robust"], maxPSCLoop=2, minFluxPSC=parms["minFluxPSC"], solPInt=parms["solPInt"], solMode="P", maxASCLoop=parms["maxASCLoop"], minFluxASC=parms["minFluxASC"], solAInt=parms["solAInt"], avgPol=parms["avgPol"], avgIF=parms["avgIF"], minSNR=parms["minSNR"], refAnt=goodCal["bestRef"], nTaper=parms["nTaper"], Tapers=parms["Tapers"], do3D=parms["do3D"], nThreads=nThreads, noScrat=noScrat, logfile=logFile, check=check, debug=debug) # Save UV data? if parms["doSaveUV"] and (not check): logger.info("--> Save UV data (doSaveUV)") mess = "Write calibrated and averaged UV data to disk" printMess(mess, logFile) # Get calibrated/averaged data if not uvc: Aname = VLBAAIPSName(project, session) uvname = project + "_" + session + "_" + band + "_Cal" uvc = UV.newPAUV(uvname, Aname, parms["avgClass"], disk, seq, True, err) if err.isErr: OErr.printErrMsg(err, "Error creating cal/avg AIPS data") # Write filename = project + session + band + "CalAvg.uvtab" fuv = VLBAUVFITS(uvc, filename, 0, err, compress=Compress) VLBAAddOutFile(filename, 'project', "Calibrated Averaged UV data") # Save list of output files VLBASaveOutFiles() # Save UV data tables? if parms["doSaveTab"] and (not check): logger.info("--> Save UV data tables (doSaveTab)") mess = "Write UV data tables to disk." printMess(mess, logFile) filename = project + session + band + "CalTab.uvtab" fuv = VLBAUVFITSTab(uv, filename, 0, err) VLBAAddOutFile(filename, 'project', "Calibrated AIPS tables") # Save list of output files VLBASaveOutFiles() # Imaging results outDisk = 0 if parms["doSaveImg"]: logger.info("--> Save images (doSaveImg)") mess = "Write images to disk." printMess(mess, logFile) # How many Stokes images nstok = len(parms["Stokes"]) # Targets logger.debug("parms['targets'] = " + str(parms['targets'])) for target in parms["targets"]: if not check: # intermediate images oclass = parms["outTclass"] x = Image.newPAImage("out", target, oclass, disk, seq, True, err) if (not x.exist): print(target, "image not found. Skipping.") continue outfile = project + session + band + target + "." + oclass + ".fits" mess = "Write Intermediate target " + outfile + " on disk " + str( outDisk) VLBAAddOutFile(outfile, target, 'Intermediate target image') printMess(mess, logFile) xf = VLBAImFITS(x, outfile, outDisk, err, fract=0.1) # Save list of output files VLBASaveOutFiles() # Statistics zz = imstat(x, err, logfile=logFile) del x, xf # Final images for istok in range(0, nstok): oclass = parms["Stokes"][istok:istok + 1] + parms["outIclass"][1:] x = Image.newPAImage("out", target, oclass, disk, seq, True, err) outfile = project + session + band + target + "." + oclass + ".fits" logger.info("Write " + outfile + " on disk " + str(outDisk)) xf = VLBAImFITS(x, outfile, outDisk, err, fract=0.1) VLBAAddOutFile(outfile, target, 'Image') logger.info("Writing file " + outfile) # Statistics zz = imstat(x, err, logfile=logFile) del x, xf # Save list of output files VLBASaveOutFiles() # Calibrators for target in parms["contCals"]: if not check: oclass = parms["outCclass"] x = Image.newPAImage("out", target, oclass, disk, seq, True, err) if (not x.exist): print(target, "image not found. Skipping.") continue outfile = project + session + band + target + "." + oclass + ".fits" mess = "Write Calibrator " + outfile + " on disk " + str( outDisk) printMess(mess, logFile) xf = VLBAImFITS(x, outfile, outDisk, err, fract=0.1) VLBAAddOutFile(outfile, target, 'Calibrator Image') # Statistics zz = imstat(x, err, logfile=logFile) del x, xf # Save list of output files VLBASaveOutFiles() # end writing images loop # Contour plots if parms["doKntrPlots"]: logger.info("--> Contour plots (doKntrPlots)") VLBAKntrPlots(err, imName=parms["targets"], project=project, session=session, band=band, disk=disk, debug=debug) # Save list of output files VLBASaveOutFiles() elif debug: print("Not creating contour plots ( doKntrPlots = ", parms["doKntrPlots"], " )") # Source uv plane diagnostic plots if parms["doDiagPlots"]: logger.info("--> Diagnostic plots (doDiagPlots)") # Get the highest number avgClass catalog file Aname = VLBAAIPSName(project, session) uvc = None if not check: uvname = project + "_" + session + "_" + band + "_Cal" uvc = UV.newPAUV(uvname, Aname, parms["avgClass"], disk, seq, True, err) VLBADiagPlots( uvc, err, cleanUp=parms["doCleanup"], \ project=project, session=session, band=band, \ logfile=logFile, check=check, debug=debug ) # Save list of output files VLBASaveOutFiles() elif debug: print("Not creating diagnostic plots ( doDiagPlots = ", parms["doDiagPlots"], " )") # Save metadata srcMetadata = None projMetadata = None if parms["doMetadata"]: logger.info("--> Save metadata (doMetadata)") if not uvc: # Get calibrated/averaged data Aname = VLBAAIPSName(project, session) uvname = project + "_" + session + "_" + band + "_Cal" uvc = UV.newPAUV(uvname, Aname, parms["avgClass"], disk, seq, True, err) if err.isErr: OErr.printErrMsg(err, "Error creating cal/avg AIPS data") # Get source metadata; save to pickle file srcMetadata = VLBASrcMetadata(uvc, err, Sources=parms["targets"], seq=seq, sclass=parms["outIclass"], Stokes=parms["Stokes"], logfile=logFile, check=check, debug=debug) picklefile = project + "_" + session + "_" + band + "SrcReport.pickle" SaveObject(srcMetadata, picklefile, True) VLBAAddOutFile(picklefile, 'project', 'All source metadata') # Get project metadata; save to pickle file projMetadata = VLBAProjMetadata(uvc, AIPS_VERSION, err, contCals=parms["contCals"], goodCal=goodCal, project=project, session=session, band=band, dataInUVF=dataInUVF, archFileID=archFileID) picklefile = project + "_" + session + "_" + band + "ProjReport.pickle" SaveObject(projMetadata, picklefile, True) VLBAAddOutFile(picklefile, 'project', 'Project metadata') # Write report if parms["doHTML"]: logger.info("--> Write HTML report (doHTML)") VLBAHTMLReport( projMetadata, srcMetadata, \ outfile=project+"_"+session+"_"+band+"report.html", \ logFile=logFile ) # Write VOTable if parms["doVOTable"]: logger.info("--> Write VOTable (doVOTable)") VLBAAddOutFile('VOTable.xml', 'project', 'VOTable report') VLBAWriteVOTable(projMetadata, srcMetadata, filename='VOTable.xml') # Save list of output files VLBASaveOutFiles() # Cleanup - delete AIPS files if parms["doCleanup"] and (not check): logger.info("--> Clean up (doCleanup)") # Delete target images # How many Stokes images nstok = len(parms["Stokes"]) for istok in range(0, nstok): oclass = parms["Stokes"][istok:istok + 1] + parms["outIclass"][1:] AllDest(err, disk=disk, Aseq=seq, Aclass=oclass) # delete Calibrator images AllDest(err, disk=disk, Aseq=seq, Aclass=parms["outCclass"]) # Delete intermediate target images AllDest(err, disk=disk, Aseq=seq, Aclass=parms["outTclass"]) OErr.printErrMsg(err, "Deleting AIPS images") # Delete UV data uv.Zap(err) # Zap calibrated/averaged data if not uvc: Aname = VLBAAIPSName(project, session) uvc = UV.newPAUV("AIPS UV DATA", Aname, parms["avgClass"], disk, seq, True, err) if err.isErr: OErr.printErrMsg(err, "Error creating cal/avg AIPS data") uvc.Zap(err) OErr.printErrMsg(err, "Writing output/cleanup") # Shutdown mess = "Finished project " + project printMess(mess, logFile) OErr.printErr(err) OSystem.Shutdown(ObitSys)
def pipeline(aipsSetup, parmFile): """ ALMA Continuum pipeline. * *aipsSetup* = AIPS setup file * *parmFile* = pipeline input parameters file """ ############################# Initialize OBIT ########################################## noScrat = [] exec(open(aipsSetup).read()) ALMAAddOutFile(aipsSetup, 'project', "Obit's AIPS setup file") ############################# Default parameters ########################################## # Initialize parameters parms = ALMAInitContParms() ############################# Set Project Processing parameters ################## print "parmFile", parmFile exec(open(parmFile).read()) ALMAAddOutFile(parmFile, 'project', 'Pipeline input parameters') # frequency/configuration dependent default parameters ALMAInitContFQParms(parms) # General data parameters band = parms["band"] # Observing band dataClass = ("UVDa" + band)[0:6] # AIPS class of raw uv data project = parms["project"][ 0:12] # Project name (12 char or less, used as AIPS Name) session = parms["session"] # Project session code ################################## Process ##################################### fileRoot = parms["project"] + "_" + parms["session"] + "_" + parms[ "band"] # root of file name logFile = fileRoot + ".log" # Processing log file uv = None uvc = None avgClass = ("UVAv" + band)[0:6] # Averaged data AIPS class outIClass = parms["outIClass"] # image AIPS class outCClass = parms["outCClass"] # image AIPS class # Load the outputs pickle jar ALMAFetchOutFiles() # Logging directly to logFile OErr.PInit(err, parms["prtLv"], logFile) OSystem.PAllowThreads(nThreads) # Allow threads in Obit/oython retCode = 0 ALMAAddOutFile(logFile, 'project', 'Pipeline log file') mess = "Start project "+parms["project"]+" session "+parms["session"]+\ " "+parms["band"]+" Band"+" AIPS user no. "+str(AIPS.userno)+\ ", ALMA Max. baseline "+str(parms["ALMAMaxBl"]) printMess(mess, logFile) if debug: pydoc.ttypager = pydoc.plainpager # don't page task input displays mess = "Using Debug mode " printMess(mess, logFile) if check: mess = "Only checking script" printMess(mess, logFile) # Log parameters if parms['doLogParms']: printMess("Parameter settings", logFile) for p in parms: mess = " " + p + ": " + str(parms[p]) printMess(mess, logFile) # Save parameters to pickle jar, manifest ParmsPicklefile = project + "_" + session + "_" + band + ".Parms.pickle" # Where results saved SaveObject(parms, ParmsPicklefile, True) ALMAAddOutFile(ParmsPicklefile, 'project', 'Processing parameters used') # Are we going to be doing Hanning? if parms["doHann"]: loadClass = parms["band"] + "Raw" else: loadClass = dataClass # Load Data from Archive directory if parms["doLoadArchive"]: uv = ALMAUVLoadArch(parms["archRoots"], ALMAAIPSName(project, session), loadClass, disk, parms["seq"], err, \ selConfig=parms["selConfig"], selBand=parms["selBand"], selChan=parms["selChan"], \ selChBW=parms["selChBW"], selNIF=parms["selNIF"], calInt=parms["calInt"], \ logfile=logFile, Compress=parms["Compress"], check=check, debug=debug) if uv == None and not check: raise RuntimeError, "Cannot load " + parms["archRoots"] # Hanning if parms["doHann"]: # Set uv if not done if uv == None and not check: uv = UV.newPAUV("AIPS UV DATA", ALMAAIPSName(project, session), loadClass[0:6], disk, parms["seq"], True, err) if err.isErr: OErr.printErrMsg(err, "Error creating AIPS data") uv = ALMAHann(uv, ALMAAIPSName(project, session), dataClass, disk, parms["seq"], err, \ doDescm=parms["doDescm"], logfile=logFile, check=check, debug=debug) if uv == None and not check: raise RuntimeError, "Cannot Hann data " # Set uv if not done if uv == None and not check: uv = UV.newPAUV("AIPS UV DATA", ALMAAIPSName(project, session), dataClass[0:6], \ disk, parms["seq"], True, err) if err.isErr: OErr.printErrMsg(err, "Error creating AIPS data") # Clear any old calibration/editing if parms["doClearTab"]: mess = "Clear previous calibration" printMess(mess, logFile) ALMAClearCal(uv, err, doGain=parms["doClearGain"], doFlag=parms["doClearFlag"], doBP=parms["doClearBP"], check=check) OErr.printErrMsg(err, "Error resetting calibration") # Copy FG 1 to FG 2 if parms["doCopyFG"]: mess = "Copy FG 1 to FG 2" printMess(mess, logFile) retCode = ALMACopyFG(uv, err, logfile=logFile, check=check, debug=debug) if retCode != 0: raise RuntimeError, "Error Copying FG table" # Drop end channels of spectra? Only if new FG 2 if parms["doCopyFG"] and (parms["BChDrop"] > 0) or (parms["EChDrop"] > 0): # Channels based on original number, reduced if Hanning nchan = uv.Desc.Dict["inaxes"][uv.Desc.Dict["jlocf"]] fact = max(1, parms["selChan"] / nchan) # Hanning reduction factor BChDrop = parms["BChDrop"] / fact EChDrop = parms["EChDrop"] / fact mess = "Trim %d channels from start and %d from end of each spectrum" % ( BChDrop, EChDrop) printMess(mess, logFile) retCode = ALMADropChan (uv, BChDrop, EChDrop, err, flagVer=parms["editFG"], \ logfile=logFile, check=check, debug=debug) if retCode != 0: raise RuntimeError, "Error Copying FG table" # Get list of source in data AllSource = ALMAAllSource(uv, err, logfile=logFile, check=check, debug=debug) # Edit source lists to remove sources not present for c in parms["PCInsCals"]: if c not in AllSource: parms["PCInsCals"].remove(c) for t in parms["targets"]: if t not in AllSource: parms["targets"].remove(t) if parms["XYGainSource"] not in AllSource: parms["XYGainSource"] = None if parms["XYDelaySource"] not in AllSource: parms["XYDelaySource"] = None for c in parms["DCals"]: if c['Source'] not in AllSource: c['Source'] = None for c in parms["BPCals"]: if c['Source'] not in AllSource: c['Source'] = None for c in parms["PCals"]: if c['Source'] not in AllSource: c['Source'] = None for c in parms["ACals"]: if c['Source'] not in AllSource: c['Source'] = None # Special editing if parms["doEditList"] and not check: mess = "Special editing" printMess(mess, logFile) for edt in parms["editList"]: UV.PFlag(uv,err,timeRange=[dhms2day(edt["timer"][0]),dhms2day(edt["timer"][1])], \ flagVer=parms["editFG"], Ants=edt["Ant"], Chans=edt["Chans"], IFs=edt["IFs"], \ Stokes=edt["Stokes"], Reason=edt["Reason"]) OErr.printErrMsg(err, "Error Flagging") # Quack to remove data from start and end of each scan if parms["doQuack"]: retCode = ALMAQuack (uv, err, begDrop=parms["quackBegDrop"], endDrop=parms["quackEndDrop"], \ Reason=parms["quackReason"], \ logfile=logFile, check=check, debug=debug) if retCode != 0: raise RuntimeError, "Error Quacking data" # Flag antennas shadowed by others? if parms["doShad"]: retCode = ALMAShadow (uv, err, shadBl=parms["shadBl"], \ logfile=logFile, check=check, debug=debug) if retCode != 0: raise RuntimeError, "Error Shadow flagging data" # Apply online calibration if parms["doOnlineCal"]: retCode = ALMAOnlineCal(uv, err, logfile=logFile, check=check, debug=debug) if retCode != 0: raise RuntimeError, "Error applying online calibration" # Median window time editing, for RFI impulsive in time if parms["doMedn"]: mess = "Median window time editing, for RFI impulsive in time:" printMess(mess, logFile) retCode = ALMAMedianFlag (uv, " ", err, noScrat=noScrat, nThreads=nThreads, \ avgTime=parms["avgTime"], avgFreq=parms["avgFreq"], chAvg= parms["chAvg"], \ timeWind=parms["timeWind"], flagVer=2,flagSig=parms["mednSigma"], \ logfile=logFile, check=check, debug=False) if retCode != 0: raise RuntimeError, "Error in MednFlag" # Median window frequency editing, for RFI impulsive in frequency if parms["doFD1"]: mess = "Median window frequency editing, for RFI impulsive in frequency:" printMess(mess, logFile) retCode = ALMAAutoFlag (uv, " ", err, flagVer=2, doCalib=-1, doBand=-1, \ timeAvg=parms["FD1TimeAvg"], \ doFD=True, FDmaxAmp=1.0e20, FDmaxV=1.0e20, FDwidMW=parms["FD1widMW"], \ FDmaxRMS=[1.0e20,0.1], FDmaxRes=parms["FD1maxRes"], \ FDmaxResBL= parms["FD1maxRes"], \ nThreads=nThreads, logfile=logFile, check=check, debug=debug) if retCode != 0: raise RuntimeError, "Error in AutoFlag" # RMS/Mean editing for calibrators if parms["doRMSAvg"]: mess = "RMS/Mean editing for calibrators:" printMess(mess, logFile) clist = ALMACombineCals(parms["ACals"], parms["PCals"], parms["DCals"]) # Calibrator list retCode = ALMAAutoFlag (uv, clist, err, flagVer=2, doCalib=-1, doBand=-1, \ RMSAvg=parms["RMSAvg"], timeAvg=parms["RMSTimeAvg"], \ nThreads=nThreads, logfile=logFile, check=check, debug=debug) if retCode != 0: raise RuntimeError, "Error in AutoFlag" # Need to find a reference antenna? See if we have saved it? if (parms["refAnt"] <= 0): refAnt = FetchObject(project + "_" + session + "_" + band + ".refAnt.pickle") if refAnt: parms["refAnt"] = refAnt # Use bandpass calibrator and center half of each spectrum if parms["refAnt"] <= 0: mess = "Find best reference antenna: run Calib on BP Cal(s) " printMess(mess, logFile) parms["refAnt"] = ALMAGetRefAnt(uv, parms["BPCals"], err, flagVer=2, \ solInt=parms["bpsolint1"], nThreads=nThreads, \ logfile=logFile, check=check, debug=debug) if err.isErr: raise RuntimeError, "Error finding reference antenna" if parms["refAnts"][0] <= 0: parms["refAnts"][0] = parms["refAnt"] mess = "Picked reference antenna " + str(parms["refAnt"]) printMess(mess, logFile) # Save it ParmsPicklefile = project + "_" + session + "_" + band + ".Parms.pickle" # Where results saved SaveObject(parms, ParmsPicklefile, True) refAntPicklefile = project + "_" + session + "_" + band + ".refAnt.pickle" # Where results saved SaveObject(parms["refAnt"], refAntPicklefile, True) # Plot Raw, edited data? if parms["doRawSpecPlot"] and parms["plotSource"]: mess = "Raw Spectral plot for: " + parms["plotSource"] printMess(mess, logFile) plotFile = "./" + fileRoot + "RawSpec.ps" retCode = ALMASpectrum(uv, parms["plotSource"], parms["plotTime"], plotFile, parms["refAnt"], err, \ flagVer=2, Stokes=["XX","YY"], doband=-1, \ check=check, debug=debug, logfile=logFile ) if retCode != 0: raise RuntimeError, "Error in Plotting spectrum" ALMAAddOutFile(plotFile, 'project', 'Pipeline log file') # delay calibration if parms["doDelayCal"] and parms["DCals"] and not check: plotFile = "./" + fileRoot + "DelayCal.ps" retCode = ALMADelayCal(uv, parms["DCals"], err, \ BChan=parms["delayBChan"], EChan=parms["delayEChan"], \ doCalib=2, flagVer=2, doBand=-1, \ solInt=parms["delaySolInt"], smoTime=parms["delaySmoo"], \ refAnts=[parms["refAnt"]], doTwo=parms["doTwo"], doZeroPhs=parms["delayZeroPhs"], \ doPlot=parms["doSNPlot"], plotFile=plotFile, \ nThreads=nThreads, noScrat=noScrat, \ logfile=logFile, check=check, debug=debug) if retCode != 0: raise RuntimeError, "Error in delay calibration" # Plot corrected data? if parms["doSpecPlot"] and parms["plotSource"]: plotFile = "./" + fileRoot + "DelaySpec.ps" retCode = ALMASpectrum(uv, parms["plotSource"], parms["plotTime"], \ plotFile, parms["refAnt"], err, \ flagVer=2, Stokes=["XX","YY"], doband=-1, \ check=check, debug=debug, logfile=logFile ) if retCode != 0: raise RuntimeError, "Error in Plotting spectrum" # Bandpass calibration if parms["doBPCal"] and parms["BPCals"]: plotFile = "./" + fileRoot + "BPCal.ps" retCode = ALMABPCal(uv, parms["BPCals"], err, noScrat=noScrat, solInt1=parms["bpsolint1"], \ solInt2=parms["bpsolint2"], solMode=parms["bpsolMode"], \ BChan1=parms["bpBChan1"], EChan1=parms["bpEChan1"], \ BChan2=parms["bpBChan2"], EChan2=parms["bpEChan2"], ChWid2=parms["bpChWid2"], \ doCenter1=parms["bpDoCenter1"], refAnt=parms["refAnt"], \ UVRange=parms["bpUVRange"], doCalib=2, gainUse=0, flagVer=2, doPlot=False, \ doAmpEdit=parms["doAmpEdit"], ampSigma=parms["ampSigma"], \ doBPPlot=parms["doBPPlot"], plotBPFile=plotFile, \ nThreads=nThreads, logfile=logFile, check=check, debug=debug) if retCode != 0: raise RuntimeError, "Error in Bandpass calibration" # Plot corrected data? if parms["doSpecPlot"] and parms["plotSource"]: plotFile = "./" + fileRoot + "BPSpec.ps" retCode = ALMASpectrum(uv, parms["plotSource"], parms["plotTime"], plotFile, \ parms["refAnt"], err, flagVer=2, Stokes=["XX","YY"], doband=1, \ check=check, debug=debug, logfile=logFile ) if retCode != 0: raise RuntimeError, "Error in Plotting spectrum" # set X/Y gains and initial calibration if parms["doXYFixGain"] and parms["XYGainSource"] and not check: mess = "Fix X/Y gain ratios" printMess(mess, logFile) retCode = ALMAXYGain(uv, err, \ XYCal=parms["XYGainSource"],timerange=parms["XYGainTime"], \ doCalib=2, gainUse=0, doBand=1, flagVer=2, refAnt=parms["refAnt"], \ nThreads=nThreads, noScrat=noScrat, logfile=logFile, \ check=check, debug=debug) if retCode != 0: raise RuntimeError, "Error in X-Y gain fix" # Self calibrate calibrators if parms["doImgCal"] and not check: mess = "SelfCalibrate/Image calibrators" printMess(mess, logFile) src = ALMACombineCals(parms["ACals"], parms["PCals"], parms["APCals"]) # Source list ALMAImageCals(uv, err, Sources=src, seq=parms["seq"], \ sclass=parms["outCClass"], doCalib=2, flagVer=2, doBand=1, FOV=parms["CalFOV"], \ maxPSCLoop=parms["maxPSCLoop"], minFluxPSC=parms["minFluxPSC"], solPInt=parms["solPInt"], \ maxASCLoop=parms["maxASCLoop"], minFluxASC=parms["minFluxASC"],\ solAInt=parms["solAInt"], avgPol=parms["avgPol"], Niter=1000, \ avgIF=parms["avgIF"], minSNR=parms["minSNR"],\ refAnt=parms["refAnt"], nThreads=nThreads, noScrat=noScrat,\ logfile=logFile, check=check, debug=debug) # Self calibrated models now available ALMAImageModel(parms["ACals"], parms["outCClass"], disk, parms["seq"], err) ALMAImageModel(parms["PCals"], parms["outCClass"], disk, parms["seq"], err) ALMAImageModel(parms["APCals"], parms["outCClass"], disk, parms["seq"], err) # Phase Calibrate if parms["doPhaseCal"]: plotFile = "./" + fileRoot + "PhaseCal.ps" retCode = ALMAPhaseCal (uv, parms["PCals"], err, ACals=parms["ACals"], \ doCalib=2, doBand=1, BPVer=1, flagVer=2, refAnt=parms["refAnt"], \ BChan=parms["ampBChan"], EChan=parms["ampEChan"], \ solInt=parms["solPInt"], avgIF = parms['avgIF'], \ ampScalar=parms["ampScalar"], doPlot=parms["doSNPlot"], plotFile=plotFile, \ nThreads=nThreads, noScrat=noScrat, logfile=logFile, check=check, debug=debug) if retCode != 0: raise RuntimeError, "Error calibrating" # Amp & phase Calibrate if parms["doAmpPhaseCal"]: plotFile = "./" + fileRoot + "APCal.ps" retCode = ALMACalAP (uv, [], parms["ACals"], err, PCals=parms["APCals"], \ doCalib=2, doBand=1, BPVer=1, flagVer=2, \ BChan=parms["ampBChan"], EChan=parms["ampEChan"], \ solInt=parms["solAInt"], solSmo=parms["solSmo"], ampScalar=parms["ampScalar"], \ doAmpEdit=parms["doAmpEdit"], ampSigma=parms["ampSigma"], \ ampEditFG=parms["ampEditFG"], \ doPlot=parms["doSNPlot"], plotFile=plotFile, refAnt=parms["refAnt"], \ nThreads=nThreads, noScrat=noScrat, logfile=logFile, check=check, debug=debug) if retCode != 0: raise RuntimeError, "Error calibrating" # More editing if parms["doAutoFlag"]: mess = "Post calibration editing:" printMess(mess, logFile) clist = [] retCode = ALMAAutoFlag (uv, clist, err, flagVer=2, \ doCalib=2, gainUse=0, doBand=1, BPVer=1, \ IClip=parms["IClip"], minAmp=parms["minAmp"], timeAvg=parms["timeAvg"], \ doFD=parms["doAFFD"], FDmaxAmp=parms["FDmaxAmp"], FDmaxV=parms["FDmaxV"], \ FDwidMW=parms["FDwidMW"], FDmaxRMS=parms["FDmaxRMS"], \ FDmaxRes=parms["FDmaxRes"], FDmaxResBL=parms["FDmaxResBL"], \ FDbaseSel=parms["FDbaseSel"], \ nThreads=nThreads, logfile=logFile, check=check, debug=debug) if retCode != 0: raise RuntimeError, "Error in AutoFlag" # Calibrate and average data if parms["doCalAvg"]: retCode = ALMACalAvg (uv, avgClass, parms["seq"], parms["CalAvgTime"], err, \ flagVer=2, doCalib=2, gainUse=0, doBand=1, BPVer=1, doPol=False, \ avgFreq=parms["CAavgFreq"], chAvg=parms["CAchAvg"], \ BChan=parms["CABChan"], EChan=parms["CAEChan"], \ BIF=parms["CABIF"], EIF=parms["CAEIF"], Compress=parms["Compress"], \ nThreads=nThreads, logfile=logFile, check=check, debug=debug) if retCode != 0: raise RuntimeError, "Error in CalAvg" # Get calibrated/averaged data if not check: try: uv = UV.newPAUV("AIPS UV DATA", ALMAAIPSName(project, session), avgClass[0:6], \ disk, parms["seq"], True, err) except Exception, exception: mess = "No Averaged/calibrated datafile" printMess(mess, logFile) return