def MKContPipeline(files, outputdir, **kwargs):
"""MeerKAT Continuum pipeline.
Parameters
----------
files : list
h5 filenames (note: support for multiple h5 files
i.e. ConcatenatedDataSet is not currently supported)
outputdir : string
Directory location to write output data,
scratchdir : string, optional
The directory location of the aips disk
parmFile : string, optional
Overwrite the default imaging parameters using this parameter file.
"""
if len(files) == 1:
h5file = files[0]
else:
h5file = files
############### Initialize katfile object #########################
OK = False
# Open the h5 file as a katfile object
try:
#open katfile and perform selection according to kwargs
katdal_ref_ant = kwargs.get('katdal_refant', '')
katdal_retries = kwargs.get('katdal_retries', 2)
katdal_timeout = kwargs.get('katdal_timeout', 300)
katdata = katfile.open(h5file,
ref_ant=katdal_ref_ant,
timeout=katdal_timeout,
retries=katdal_retries)
OK = True
except Exception as exception:
print(exception)
if not OK:
raise KATUnimageableError("Unable to read MVF data in " + str(h5file))
# If we are doing polcal- search for the most recent delaycal observation
if kwargs.get('polcal'):
if kwargs.get('delaycal_mvf') is None:
# Automatically determine delay_cal CBID
delay_katdata = KATGetDelayCal(h5file,
katdata,
timeout=katdal_timeout,
retries=katdal_retries)
else:
# Use the user supplied one
delay_katdata = KATGetDelayCal(kwargs.get('delaycal_mvf'))
kwargs["delay_katdata"] = delay_katdata
# Die gracefully if we cannot write to the output area...
if not os.path.exists(outputdir):
print('Specified output directory: ' + outputdir + 'does not exist.')
exit(-1)
# Obit error logging
err = OErr.OErr()
#################### Initialize filenames #######################################################
nameRoot = katdata.obs_params.get('capture_block_id',
katdata.experiment_id)
if type(nameRoot) == list:
nameRoot = nameRoot[0]
fileRoot = os.path.join(outputdir, nameRoot) # root of file name
logFile = fileRoot + ".log" # Processing log file
avgClass = ("UVAv")[0:6] # Averaged data AIPS class
manifestfile = outputdir + '/manifest.pickle'
############################# Initialize OBIT and AIPS ##########################################
noScrat = []
# Logging directly to logFile
OErr.PInit(err, 2, logFile)
EVLAAddOutFile(os.path.basename(logFile), 'project', 'Pipeline log file')
if kwargs.get('reuse'):
ObitSys = AIPSSetup.AIPSSetup(err,
configfile=kwargs.get('configFile'),
scratchdir=kwargs.get('scratchdir'),
aipsdisk=kwargs.get('aipsdisk'),
overwrite=False)
else:
ObitSys = AIPSSetup.AIPSSetup(err,
configfile=kwargs.get('configFile'),
scratchdir=kwargs.get('scratchdir'),
aipsdisk=kwargs.get('aipsdisk'))
# Get the set up AIPS environment.
AIPS_ROOT = os.environ['AIPS_ROOT']
AIPS_VERSION = os.environ['AIPS_VERSION']
nThreads = 72
user = OSystem.PGetAIPSuser()
AIPS.userno = user
disk = 1
fitsdisk = 1
nam = nameRoot[:10]
clss = "Raw"
seq = 1
############### Condition data #########################
#Get calibrator models
fluxcals = katpoint.Catalogue(
open(FITSDir.FITSdisks[1] + "/PERLEY_BUTLER_2013.csv"))
#Condition data (get bpcals, update names for aips conventions etc)
KATh5Condition(katdata, fluxcals, err)
############################# Initialise Parameters ##########################################
####### Initialize parameters dictionary #####
parms = KATInitContParms()
parms['PolCal'] = kwargs.get('polcal')
parms['XYtarg'] = kwargs.get('XYtarg')
# Get default XYtarg if it is not set
targs = [targ.name for targ in katdata.catalogue.targets]
if parms['PolCal']:
if parms['XYtarg'] is None:
GOTTARG = False
for targ in ['1934-638', '0408-65']:
if targ in targs:
parms['XYtarg'] = targ
GOTTARG = True
break
if not GOTTARG:
raise RuntimeError(
'No default targets (1934-638, 0408-65) for XYFix. Cannot run in PolCal mode.'
)
else:
if parms['XYtarg'] not in targs:
raise RuntimeError(
'XYtarg target %s not in observation. Cannot run in PolCal mode.'
% (parms['XYtarg']))
####### User defined parameters ######
if kwargs.get('parmFile'):
print("parmFile", kwargs.get('parmFile'))
exec(open(kwargs.get('parmFile')).read())
EVLAAddOutFile(os.path.basename(kwargs.get('parmFile')), 'project',
'Pipeline input parameters')
###################### Data selection and static edits ############################################
# Select data based on static imageable parameters
KATh5Select(katdata, parms, err, **kwargs)
# General AIPS data parameters at script level
dataClass = ("UVDa")[0:6] # AIPS class of raw uv data
delayClass = "DELA"
band = katdata.spectral_windows[0].product #Correlator product
project = os.path.basename(os.path.splitext(files[0])[0])[
0:10] # Project name (12 char or less, used as AIPS Name)
outIClass = parms["outIClass"] # image AIPS class
debug = parms["debug"]
check = parms["check"]
####################### Import data into AIPS #####################################################
# Reuse or nay?
sw = katdata.spectral_windows[katdata.spw]
# Pick up static flags
if sw.band == 'L':
sflags = FetchObject(ObitTalkUtil.FITSDir.FITSdisks[fitsdisk] +
'maskred.pickle')
if kwargs.get('flag', None):
mess = 'Using static RFI mask in file %s for L-band' % (
ObitTalkUtil.FITSDir.FITSdisks[fitsdisk] + 'maskred.pickle', )
printMess(mess, logFile)
elif sw.band == 'UHF':
sflags = FetchObject(ObitTalkUtil.FITSDir.FITSdisks[fitsdisk] +
'maskredUHF.pickle')
if kwargs.get('flag', None):
mess = 'Using static RFI mask in file %s for UHF-band' % (
ObitTalkUtil.FITSDir.FITSdisks[fitsdisk] +
'maskredUHF.pickle', )
printMess(mess, logFile)
else:
sflags = np.zeros(sw.num_chans, dtype=np.bool)
sflags = sflags[katdata.channels]
# Construct a template uvfits file from master template
mastertemplate = ObitTalkUtil.FITSDir.FITSdisks[
fitsdisk] + 'MKATTemplate.uvtab.gz'
outtemplate = nam + '.uvtemp'
if kwargs.get('reuse'):
uv = UV.newPAUV("AIPS UV DATA", EVLAAIPSName(project), dataClass, disk,
seq, True, err)
obsdata = KATH5toAIPS.GetKATMeta(katdata, err)
# Extract AIPS parameters of the uv data to the metadata
obsdata["Aproject"] = uv.Aname
obsdata["Aclass"] = uv.Aclass
obsdata["Aseq"] = uv.Aseq
obsdata["Adisk"] = disk
obsdata["calInt"] = katdata.dump_period
obsdata["fitsdisk"] = fitsdisk
# TODO: Check if the input data has been Hanned.
doneHann = True
else:
mess = '\nLoading UV data with CBID: %s' % (
katdata.obs_params['capture_block_id'], )
printMess(mess, logFile)
KATH5toAIPS.MakeTemplate(mastertemplate, outtemplate, katdata)
uv = OTObit.uvlod(outtemplate, 0, EVLAAIPSName(project), clss, disk,
seq, err)
obsdata = KATH5toAIPS.KAT2AIPS(katdata,
uv,
disk,
fitsdisk,
err,
calInt=katdata.dump_period,
static=sflags,
**kwargs)
MakeIFs.UVMakeIF(uv, 8, err, solInt=katdata.dump_period)
os.remove(outtemplate)
if parms["PolCal"]:
mess = '\nLoading delay calibration with CBID: %s' % (
delay_katdata.obs_params['capture_block_id'], )
printMess(mess, logFile)
# Load the delay cal observation
KATH5toAIPS.MakeTemplate(mastertemplate, outtemplate, katdata)
delay_uv = OTObit.uvlod(outtemplate, 0, EVLAAIPSName(project),
delayClass, disk, seq, err)
KATH5toAIPS.KAT2AIPS(delay_katdata,
delay_uv,
disk,
fitsdisk,
err,
calInt=katdata.dump_period,
static=sflags,
flag=False)
MakeIFs.UVMakeIF(delay_uv, 8, err, solInt=katdata.dump_period)
os.remove(outtemplate)
# Print the uv data header to screen.
uv.Header(err)
############################# Set Project Processing parameters ###################################
# Parameters derived from obsdata and katdata
KATGetObsParms(obsdata, katdata, parms, logFile)
###### Initialise target parameters #####
KATInitTargParms(katdata, parms, err)
# Load the outputs pickle jar
EVLAFetchOutFiles()
OSystem.PAllowThreads(nThreads) # Allow threads in Obit/oython
retCode = 0
doBand = -1
BPVer = 0
maxgap = max(parms["CalAvgTime"], 160. * katdata.dump_period) / 60.
################### Start processing ###############################################################
mess = "Start project "+parms["project"]+" AIPS user no. "+str(AIPS.userno)+\
", KAT7 configuration "+parms["KAT7Cfg"]
printMess(mess, logFile)
if debug:
pydoc.ttypager = pydoc.plainpager # don't page task input displays
mess = "Using Debug mode "
printMess(mess, logFile)
if check:
mess = "Only checking script"
printMess(mess, logFile)
# Log parameters
printMess("Parameter settings", logFile)
for p in parms:
mess = " " + p + ": " + str(parms[p])
printMess(mess, logFile)
clist = []
for DCal in parms["DCals"]:
if DCal["Source"] not in clist:
clist.append(DCal["Source"])
for PCal in parms["PCals"]:
if PCal["Source"] not in clist:
clist.append(PCal["Source"])
for ACal in parms["ACals"]:
if ACal["Source"] not in clist:
clist.append(ACal["Source"])
if kwargs.get('targets') is not None:
targets = [
targ.name for targ in katdata.catalogue
if (targ.name not in clist) and (
targ.name in kwargs.get('targets').split(','))
]
else:
targets = [
targ.name for targ in katdata.catalogue if (targ.name not in clist)
]
refAnt = kwargs.get('refant')
if refAnt is not None:
try:
SaveObject(obsdata['antLookup'][refAnt],
fileRoot + ".refAnt.pickle", True)
except:
mess = "Select reference antenna " + refAnt + " not in antenna table."
printMess(mess, logFile)
print(mess)
refAnt = FetchObject(fileRoot + ".refAnt.pickle")
# Save parameters to pickle jar, manifest
ParmsPicklefile = fileRoot + ".Parms.pickle" # Where results saved
SaveObject(parms, ParmsPicklefile, True)
EVLAAddOutFile(os.path.basename(ParmsPicklefile), 'project',
'Processing parameters used')
loadClass = dataClass
# Hanning - only if not reusing
doneHann = False
if not kwargs.get('reuse'):
if parms["doHann"]:
uv = KATHann(uv, EVLAAIPSName(project), dataClass, disk, seq, err, \
doDescm=parms["doDescm"], flagVer=-1, logfile=logFile, zapin=True, check=check, debug=debug)
doneHann = True
if parms["PolCal"] and parms["doHann"]:
mess = "Hanning delay calibration scan"
printMess(mess, logFile)
delay_uv = KATHann(delay_uv, EVLAAIPSName(project), delayClass, disk, seq + 1, err, \
doDescm=parms["doDescm"], flagVer=-1, logfile=logFile, zapin=True, check=check, debug=debug)
if doneHann:
# Halve channels after hanning.
parms["selChan"] = int(parms["selChan"] / 2)
parms["BChDrop"] = int(parms["BChDrop"] / 2)
parms["EChDrop"] = int(parms["EChDrop"] / 2)
if uv == None and not check:
raise RuntimeError("Cannot Hann data ")
# Clear any old calibration/editing
if parms["doClearTab"] or kwargs.get('reuse'):
mess = "Clear previous calibration"
printMess(mess, logFile)
EVLAClearCal(uv,
err,
doGain=parms["doClearGain"],
doFlag=parms["doClearFlag"],
doBP=parms["doClearBP"],
check=check)
OErr.printErrMsg(err, "Error resetting calibration")
# Copy FG 1 to FG 2
if parms["doCopyFG"]:
mess = "Copy FG 1 to FG 2"
printMess(mess, logFile)
retCode = KATCopyFG(uv, err, logfile=logFile, check=check, debug=debug)
if retCode != 0:
raise RuntimeError("Error Copying FG table")
# Flag antennas shadowed by others?
if parms["doShad"]:
retCode = EVLAShadow (uv, err, shadBl=parms["shadBl"], \
logfile=logFile, check=check, debug=debug)
if retCode != 0:
raise RuntimeError("Error Shadow flagging data")
# Median window time editing, for RFI impulsive in time
if parms["doMednTD1"]:
mess = "Median window time editing, for RFI impulsive in time:"
printMess(mess, logFile)
retCode = EVLAMedianFlag (uv, clist, err, noScrat=noScrat, nThreads=nThreads, \
avgTime=parms["mednAvgTime"], avgFreq=parms["mednAvgFreq"], chAvg= parms["mednChAvg"], \
timeWind=parms["mednTimeWind"],flagVer=2, flagTab=2,flagSig=parms["mednSigma"], \
logfile=logFile, check=check, debug=False)
if retCode != 0:
raise RuntimeError("Error in MednFlag")
# Median window frequency editing, for RFI impulsive in frequency
if parms["doFD1"]:
mess = "Median window frequency editing, for RFI impulsive in frequency:"
printMess(mess, logFile)
retCode = EVLAAutoFlag (uv, clist, err, flagVer=2, flagTab=2, doCalib=-1, doBand=-1, \
timeAvg=parms["FD1TimeAvg"], \
doFD=True, FDmaxAmp=1.0e20, FDmaxV=1.0e20, FDwidMW=parms["FD1widMW"], \
FDmaxRMS=[1.0e20,0.1], FDmaxRes=parms["FD1maxRes"], \
FDmaxResBL= parms["FD1maxRes"], FDbaseSel=parms["FD1baseSel"],\
nThreads=nThreads, logfile=logFile, check=check, debug=debug)
if retCode != 0:
raise RuntimeError("Error in AutoFlag")
# Parallactic angle correction?
if parms["doPACor"]:
retCode = EVLAPACor(uv, err, \
logfile=logFile, check=check, debug=debug)
if retCode != 0:
raise RuntimeError("Error in Parallactic angle correction")
# Need to find a reference antenna? See if we have saved it?
if (parms["refAnt"] <= 0):
refAnt = FetchObject(fileRoot + ".refAnt.pickle")
if refAnt:
parms["refAnt"] = refAnt
# Use bandpass calibrator and center half of each spectrum
if parms["refAnt"] <= 0:
mess = "Find best reference antenna: run Calib on BP Cal(s) "
printMess(mess, logFile)
parms["refAnt"] = EVLAGetRefAnt(uv, parms["BPCals"], err, flagVer=0, \
solInt=parms["bpsolint1"], nThreads=nThreads, \
logfile=logFile, check=check, debug=debug)
if err.isErr:
raise RuntimeError("Error finding reference antenna")
if parms["refAnts"][0] <= 0:
parms["refAnts"][0] = parms["refAnt"]
mess = "Picked reference antenna " + str(parms["refAnt"])
printMess(mess, logFile)
# Save it
ParmsPicklefile = fileRoot + ".Parms.pickle" # Where results saved
SaveObject(parms, ParmsPicklefile, True)
refAntPicklefile = fileRoot + ".refAnt.pickle" # Where results saved
SaveObject(parms["refAnt"], refAntPicklefile, True)
# Plot Raw, edited data?
if parms["doRawSpecPlot"] and parms["plotSource"]:
mess = "Raw Spectral plot for: " + ' '.join(parms["BPCal"])
printMess(mess, logFile)
plotFile = fileRoot + "_RawSpec.ps"
retCode = EVLASpectrum(uv, parms["BPCal"], parms["plotTime"], maxgap, plotFile, parms["refAnt"], err, \
Stokes=["RR","LL"], doband=-1, \
check=check, debug=debug, logfile=logFile )
if retCode != 0:
raise RuntimeError("Error in Plotting spectrum")
EVLAAddOutFile(plotFile, 'project', 'Pipeline log file')
if parms["PolCal"]:
mess = "XYphase bandpass calibration"
printMess(mess, logFile)
retCode = KATXPhase(delay_uv,
uv,
err,
logfile=logFile,
check=check,
debug=debug,
doCalib=-1,
flagVer=0,
doBand=-1,
refAnt=parms['refAnt'])
doBand = 1
BPVer += 1
if retCode != 0:
raise RuntimeError("Error in Xphase calibration")
# delay calibration
if parms["doDelayCal"] and parms["DCals"] and not check:
plotFile = fileRoot + "_DelayCal.ps"
retCode = EVLADelayCal(uv, parms["DCals"], err, \
BChan=parms["delayBChan"], EChan=parms["delayEChan"], \
doCalib=-1, flagVer=0, doBand=doBand, BPVer=BPVer, \
solInt=parms["delaySolInt"], smoTime=parms["delaySmoo"], \
refAnts=[parms["refAnt"]], doTwo=parms["doTwo"],
doZeroPhs=parms["delayZeroPhs"], \
doAvgIF=parms["delayAvgIF"], doAvgPol=parms["delayAvgPol"], \
doPlot=parms["doSNPlot"], plotFile=plotFile, \
nThreads=nThreads, noScrat=noScrat, \
logfile=logFile, check=check, debug=debug)
if retCode != 0:
raise RuntimeError("Error in delay calibration")
# Plot corrected data?
if parms["doSpecPlot"] and parms["plotSource"]:
plotFile = fileRoot + "_DelaySpec.ps"
retCode = EVLASpectrum(uv, parms["BPCal"], parms["plotTime"], maxgap, \
plotFile, parms["refAnt"], err, \
Stokes=["RR","LL"], doband=doBand, \
check=check, debug=debug, logfile=logFile )
if retCode != 0:
raise RuntimeError("Error in Plotting spectrum")
# Bandpass calibration
if parms["doBPCal"] and parms["BPCals"]:
retCode = KATBPCal(uv, parms["BPCals"], err, doBand=doBand, BPVer=BPVer, newBPVer=0,
noScrat=noScrat, solInt1=parms["bpsolint1"], \
solInt2=parms["bpsolint2"], solMode=parms["bpsolMode"], \
BChan1=parms["bpBChan1"], EChan1=parms["bpEChan1"], \
BChan2=parms["bpBChan2"], EChan2=parms["bpEChan2"], ChWid2=parms["bpChWid2"], \
doCenter1=parms["bpDoCenter1"], refAnt=parms["refAnt"], \
UVRange=parms["bpUVRange"], doCalib=2, gainUse=0, flagVer=0, doPlot=False, \
nThreads=nThreads, logfile=logFile, check=check, debug=debug)
if retCode != 0:
raise RuntimeError("Error in Bandpass calibration")
# Plot corrected data?
if parms["doSpecPlot"] and parms["plotSource"]:
plotFile = fileRoot + "_BPSpec.ps"
retCode = EVLASpectrum(uv, parms["BPCal"], parms["plotTime"], maxgap, plotFile, \
parms["refAnt"], err, Stokes=["RR","LL"], doband=1, \
check=check, debug=debug, logfile=logFile )
if retCode != 0:
raise RuntimeError("Error in Plotting spectrum")
# Amp & phase Calibrate
if parms["doAmpPhaseCal"]:
plotFile = fileRoot + "_APCal.ps"
retCode = KATCalAP (uv, [], parms["ACals"], err, PCals=parms["PCals"],
doCalib=2, doBand=1, BPVer=0, flagVer=0, \
BChan=parms["ampBChan"], EChan=parms["ampEChan"], \
solInt=parms["solInt"], solSmo=parms["solSmo"], ampScalar=parms["ampScalar"], \
doAmpEdit=parms["doAmpEdit"], ampSigma=parms["ampSigma"], \
ampEditFG=parms["ampEditFG"], avgPol=parms["PolCal"], \
doPlot=parms["doSNPlot"], plotFile=plotFile, refAnt=parms["refAnt"], \
nThreads=nThreads, noScrat=noScrat, logfile=logFile, check=check, debug=debug)
if retCode != 0:
raise RuntimeError("Error calibrating")
# More editing
if parms["doAutoFlag"]:
mess = "Post calibration editing:"
printMess(mess, logFile)
# if going to redo then only calibrators
if parms["doRecal"]:
# Only calibrators
clist = []
for DCal in parms["DCals"]:
if DCal["Source"] not in clist:
clist.append(DCal["Source"])
for PCal in parms["PCals"]:
if PCal["Source"] not in clist:
clist.append(PCal["Source"])
for ACal in parms["ACals"]:
if ACal["Source"] not in clist:
clist.append(ACal["Source"])
else:
clist = []
retCode = EVLAAutoFlag (uv, clist, err, flagVer=0, flagTab =2, \
doCalib=2, gainUse=0, doBand=1, BPVer=BPVer, \
IClip=parms["IClip"], minAmp=parms["minAmp"], timeAvg=parms["timeAvg"], \
doFD=parms["doFirstAFFD"], FDmaxAmp=parms["FDmaxAmp"], FDmaxV=parms["FDmaxV"], \
FDwidMW=parms["FDwidMW"], FDmaxRMS=parms["FDmaxRMS"], \
FDmaxRes=parms["FDmaxRes"], FDmaxResBL=parms["FDmaxResBL"], \
FDbaseSel=parms["FDbaseSel"], \
nThreads=nThreads, logfile=logFile, check=check, debug=debug)
if retCode != 0:
raise RuntimeError("Error in AutoFlag")
# Redo the calibration using new flagging?
if parms["doBPCal2"] == None:
parms["doBPCal2"] = parms["doBPCal"]
if parms["doDelayCal2"] == None:
parms["doDelayCal2"] = parms["doDelayCal2"]
if parms["doAmpPhaseCal2"] == None:
parms["doAmpPhaseCal2"] = parms["doAmpPhaseCal"]
if parms["doAutoFlag2"] == None:
parms["doAutoFlagCal2"] = parms["doAutoFlag"]
if parms["doRecal"]:
mess = "Redo calibration:"
printMess(mess, logFile)
EVLAClearCal(uv,
err,
doGain=True,
doFlag=False,
doBP=True,
check=check,
logfile=logFile)
OErr.printErrMsg(err, "Error resetting calibration")
BPVer = 0
# Parallactic angle correction?
if parms["doPACor"]:
retCode = EVLAPACor(uv, err, \
logfile=logFile, check=check, debug=debug)
if retCode != 0:
raise RuntimeError("Error in Parallactic angle correction")
# Run MKXPhase on delaycal data and attach BP table to UV data
if parms["PolCal"]:
mess = "XYphase bandpass calibration"
printMess(mess, logFile)
retCode = KATXPhase(delay_uv,
uv,
err,
logfile=logFile,
check=check,
debug=debug,
doCalib=-1,
flagVer=0,
doBand=-1,
refAnt=parms['refAnt'])
BPVer += 1
if retCode != 0:
raise RuntimeError("Error in Xphase calibration")
# Delay recalibration
if parms["doDelayCal2"] and parms["DCals"] and not check:
plotFile = fileRoot + "_DelayCal2.ps"
retCode = EVLADelayCal(uv, parms["DCals"], err, \
BChan=parms["delayBChan"], EChan=parms["delayEChan"], \
doCalib=-1, flagVer=0, doBand=doBand, BPVer=BPVer, \
solInt=parms["delaySolInt"], smoTime=parms["delaySmoo"], \
refAnts=[parms["refAnt"]], doTwo=parms["doTwo"], \
doZeroPhs=parms["delayZeroPhs"], \
doAvgIF=parms["delayAvgIF"], doAvgPol=parms["delayAvgPol"], \
doPlot=parms["doSNPlot"], plotFile=plotFile, \
nThreads=nThreads, noScrat=noScrat, \
logfile=logFile, check=check, debug=debug)
if retCode != 0:
raise RuntimeError("Error in delay calibration")
# Plot corrected data?
if parms["doSpecPlot"] and parms["plotSource"]:
plotFile = fileRoot + "_DelaySpec2.ps"
retCode = EVLASpectrum(uv, parms["BPCal"], parms["plotTime"], maxgap, plotFile, parms["refAnt"], err, \
Stokes=["RR","LL"], doband=doband, \
check=check, debug=debug, logfile=logFile )
if retCode != 0:
raise RuntimeError("Error in Plotting spectrum")
# Bandpass calibration
if parms["doBPCal2"] and parms["BPCals"]:
retCode = KATBPCal(uv, parms["BPCals"], err, doBand=doBand, BPVer=BPVer, newBPVer=0, \
noScrat=noScrat, solInt1=parms["bpsolint1"], \
solInt2=parms["bpsolint2"], solMode=parms["bpsolMode"], \
BChan1=parms["bpBChan1"], EChan1=parms["bpEChan1"], \
BChan2=parms["bpBChan2"], EChan2=parms["bpEChan2"], ChWid2=parms["bpChWid2"], \
doCenter1=parms["bpDoCenter1"], refAnt=parms["refAnt"], \
UVRange=parms["bpUVRange"], doCalib=2, gainUse=0, flagVer=0, doPlot=False, \
nThreads=nThreads, logfile=logFile, check=check, debug=debug)
if retCode != 0:
raise RuntimeError("Error in Bandpass calibration")
# Plot corrected data?
if parms["doSpecPlot"] and parms["plotSource"]:
plotFile = fileRoot + "_BPSpec2.ps"
retCode = EVLASpectrum(uv, parms["BPCal"], parms["plotTime"], maxgap, plotFile, parms["refAnt"], err, \
Stokes=["RR","LL"], doband=1, \
check=check, debug=debug, logfile=logFile )
if retCode != 0:
raise RuntimeError("Error in Plotting spectrum")
# Amp & phase Recalibrate
if parms["doAmpPhaseCal2"]:
plotFile = fileRoot + "_APCal2.ps"
retCode = KATCalAP (uv, [], parms["ACals"], err, PCals=parms["PCals"], \
doCalib=2, doBand=1, BPVer=0, flagVer=0, \
BChan=parms["ampBChan"], EChan=parms["ampEChan"], \
solInt=parms["solInt"], solSmo=parms["solSmo"], ampScalar=parms["ampScalar"], \
doAmpEdit=True, ampSigma=parms["ampSigma"], \
ampEditFG=parms["ampEditFG"], avgPol=parms["PolCal"], \
doPlot=parms["doSNPlot"], plotFile=plotFile, refAnt=parms["refAnt"], \
noScrat=noScrat, nThreads=nThreads, logfile=logFile, check=check, debug=debug)
if retCode != 0:
raise RuntimeError("Error calibrating")
# More editing
if parms["doAutoFlag2"]:
mess = "Post recalibration editing:"
printMess(mess, logFile)
retCode = EVLAAutoFlag (uv, [], err, flagVer=0, flagTab=2, \
doCalib=2, gainUse=0, doBand=1, BPVer=0, \
IClip=parms["IClip"], minAmp=parms["minAmp"], timeAvg=parms["timeAvg"], \
doFD=parms["doSecAFFD"], FDmaxAmp=parms["FDmaxAmp"], FDmaxV=parms["FDmaxV"], \
FDwidMW=parms["FDwidMW"], FDmaxRMS=parms["FDmaxRMS"], \
FDmaxRes=parms["FDmaxRes"], FDmaxResBL= parms["FDmaxResBL"], \
FDbaseSel=parms["FDbaseSel"], \
nThreads=nThreads, logfile=logFile, check=check, debug=debug)
if retCode != 0:
raise RuntimeError("Error in AutoFlag")
# end recal
# Calibrate and average data
# Overwrite avgStokes from command line
if kwargs.get('halfstokes'):
parms["avgStokes"] = 'HALF'
if parms["doCalAvg"] == 'Splat':
retCode = KATCalAvg (uv, avgClass, parms["seq"], parms["CalAvgTime"], err, \
flagVer=2, doCalib=2, gainUse=0, doBand=1, BPVer=0, doPol=False, \
avgFreq=parms["avgFreq"], chAvg=parms["chAvg"], Stokes=parms["avgStokes"], \
BChan=1, EChan=parms["selChan"] - 1, doAuto=parms["doAuto"], \
BIF=parms["CABIF"], EIF=parms["CAEIF"], Compress=parms["Compress"], \
nThreads=nThreads, logfile=logFile, check=check, debug=debug)
if retCode != 0:
raise RuntimeError("Error in CalAvg")
elif parms["doCalAvg"] == 'BL':
retCode = KATBLCalAvg (uv, avgClass, parms["seq"], err, \
flagVer=2, doCalib=2, gainUse=0, doBand=1, BPVer=0, doPol=False, \
avgFreq=parms["avgFreq"], chAvg=parms["chAvg"], FOV=parms['FOV'], \
maxInt=min(parms["solPInt"],parms["solAInt"]), Stokes=parms["avgStokes"], \
BChan=1, EChan=parms["selChan"] - 1, timeAvg=parms["CalAvgTime"], \
BIF=parms["CABIF"], EIF=parms["CAEIF"], Compress=parms["Compress"], \
logfile=logFile, check=check, debug=debug)
if retCode != 0:
raise RuntimeError("Error in BLCalAvg")
if parms["doSaveTab"]:
filename = project + ".CalTab.uvtab"
_ = EVLAUVFITSTab(uv, filename, 0, err, logfile=logFile)
#Zap unaveraged data if requested
if kwargs.get('zapraw'):
uv.Zap(err)
# Get calibrated/averaged data
if not check:
uv = UV.newPAUV("AIPS UV DATA", EVLAAIPSName(project), avgClass[0:6], \
disk, parms["seq"], True, err)
if err.isErr:
OErr.printErrMsg(err, "Error creating cal/avg AIPS data")
plotFile = fileRoot + "_Spec.ps"
retCode = EVLASpectrum(uv, parms["BPCal"], parms["plotTime"], maxgap, \
plotFile, parms["refAnt"], err, \
Stokes=["I"], doband=-1, docalib=-1, \
check=check, debug=debug, logfile=logFile )
if retCode != 0:
raise RuntimeError("Error in Plotting spectrum")
# KATUVFITS(uv, 'preimage.uvfits', 0, err, exclude=["AIPS HI", "AIPS SL", "AIPS PL"],
# include=["AIPS AN", "AIPS FQ"], compress=parms["Compress"], logfile=logFile)
KATUVFITab(uv, project + '.uvtab', 0, err)
#Gzip the data?
if kwargs.get('gzip'):
os.system('pigz -p %d %s' % (nThreads, project + '.uvtab'))
os.system('rm -f %s' % (project + '.uvtab'))
def pipeline(scriptName, aipsSetup, parmFile):
"""
Linear (feed) Polarization Continuum pipeline.
* *scriptName* = this script file name
* *aipsSetup* = AIPS setup file
* *parmFile* = pipeline input parameters file
"""
############################# Initialize OBIT ##########################################
noScrat = []
exec(open(aipsSetup).read())
EVLAAddOutFile(aipsSetup, 'project', "Obit's AIPS setup file")
############################# Default parameters ##########################################
# Initialize parameters
parms = EVLAInitContParms()
# Linear feed specific parameters
# Relative gain calibration
parms["doXYRelGain"] = False # constrain relative X, Y gain amplitudes
parms[
"doXYRelGain2"] = None # constrain relative X, Y gain amplitudes, 2nd pass
parms["XYRelGainCal"] = None # X/Y gain calibrator structure
parms["XYRelGainCalTime"] = [0., 0.
] # Time range for relative gain calibration
############################# Set Project Processing parameters ##################
print("parmFile", parmFile)
exec(open(parmFile).read())
EVLAAddOutFile(parmFile, 'project', 'Pipeline input parameters')
# frequency/configuration dependent default parameters
EVLAInitContFQParms(parms)
# Linear feed specific:
parms["avgPol"] = parms["doXYRelGain"] and (parms["XYRelGainCal"] != None
) # Fix relative X/Y gains?
# General data parameters
band = parms["band"] # Observing band
dataClass = ("UVDa" + band)[0:6] # AIPS class of raw uv data
project = parms["project"][
0:12] # Project name (12 char or less, used as AIPS Name)
session = parms["session"] # Project session code
################################## Process #####################################
fileRoot = parms["project"] + "_" + parms["session"] + "_" + parms[
"band"] # root of file name
logFile = fileRoot + ".log" # Processing log file
uv = None
uvc = None
avgClass = ("UVAv" + band)[0:6] # Averaged data AIPS class
outIClass = parms["outIClass"] # image AIPS class
# Load the outputs pickle jar
EVLAFetchOutFiles()
# Logging directly to logFile
OErr.PInit(err, parms["prtLv"], logFile)
OSystem.PAllowThreads(nThreads) # Allow threads in Obit/oython
retCode = 0
EVLAAddOutFile(logFile, 'project', 'Pipeline log file')
mess = "Start project "+parms["project"]+" session "+parms["session"]+\
" "+parms["band"]+" Band"+" AIPS user no. "+str(AIPS.userno)+\
", EVLA configuration "+parms["VLACfg"]
printMess(mess, logFile)
if debug:
pydoc.ttypager = pydoc.plainpager # don't page task input displays
mess = "Using Debug mode "
printMess(mess, logFile)
if check:
mess = "Only checking script"
printMess(mess, logFile)
# Log parameters
printMess("Parameter settings", logFile)
for p in parms:
mess = " " + p + ": " + str(parms[p])
printMess(mess, logFile)
# Save parameters to pickle jar, manifest
ParmsPicklefile = project + "_" + session + "_" + band + ".Parms.pickle" # Where results saved
SaveObject(parms, ParmsPicklefile, True)
EVLAAddOutFile(ParmsPicklefile, 'project', 'Processing parameters used')
# Are we going to be doing Hanning?
if parms["doHann"]:
loadClass = parms["band"] + "Raw"
else:
loadClass = dataClass
# Load Data from Archive directory
if parms["doLoadArchive"]:
uv = EVLAUVLoadArch(parms["archRoot"], EVLAAIPSName(project, session), loadClass, disk, parms["seq"], err, \
selConfig=parms["selConfig"], doSwPwr=parms["doSwPwr"], \
selBand=parms["selBand"], selChan=parms["selChan"], \
selNIF=parms["selNIF"], calInt=parms["calInt"], \
logfile=logFile, Compress=parms["Compress"], check=check, debug=debug)
if uv == None and not check:
raise RuntimeError("Cannot load " + parms["DataRoot"])
# Hanning
if parms["doHann"]:
# Set uv if not done
if uv == None and not check:
uv = UV.newPAUV("AIPS UV DATA", EVLAAIPSName(project, session),
loadClass[0:6], disk, parms["seq"], True, err)
if err.isErr:
OErr.printErrMsg(err, "Error creating AIPS data")
uv = EVLAHann(uv, EVLAAIPSName(project, session), dataClass, disk, parms["seq"], err, \
doDescm=parms["doDescm"], logfile=logFile, check=check, debug=debug)
if uv == None and not check:
raise RuntimeError("Cannot Hann data ")
# Set uv if not done
if uv == None and not check:
uv = UV.newPAUV("AIPS UV DATA", EVLAAIPSName(project, session), dataClass[0:6], \
disk, parms["seq"], True, err)
if err.isErr:
OErr.printErrMsg(err, "Error creating AIPS data")
# Save file names in history
EVLAScriptHistory(uv, scriptName, aipsSetup, parmFile, err)
if err.isErr:
OErr.printErrMsg(err, "Error writing file names to history")
# Clear any old calibration/editing
if parms["doClearTab"]:
mess = "Clear previous calibration"
printMess(mess, logFile)
EVLAClearCal(uv,
err,
doGain=parms["doClearGain"],
doFlag=parms["doClearFlag"],
doBP=parms["doClearBP"],
check=check)
OErr.printErrMsg(err, "Error resetting calibration")
# Copy FG 1 to FG 2
if parms["doCopyFG"]:
mess = "Copy FG 1 to FG 2"
printMess(mess, logFile)
retCode = EVLACopyFG(uv,
err,
logfile=logFile,
check=check,
debug=debug)
if retCode != 0:
raise RuntimeError("Error Copying FG table")
# Drop end channels of spectra? Only if new FG 2
if parms["doCopyFG"] and (parms["BChDrop"] > 0) or (parms["EChDrop"] > 0):
# Channels based on original number, reduced if Hanning
nchan = uv.Desc.Dict["inaxes"][uv.Desc.Dict["jlocf"]]
fact = parms["selChan"] / nchan # Hanning reduction factor
BChDrop = parms["BChDrop"] / fact
EChDrop = parms["EChDrop"] / fact
mess = "Trim %d channels from start and %d from end of each spectrum" % (
BChDrop, EChDrop)
printMess(mess, logFile)
retCode = EVLADropChan (uv, BChDrop, EChDrop, err, flagVer=parms["editFG"], \
logfile=logFile, check=check, debug=debug)
if retCode != 0:
raise RuntimeError("Error Copying FG table")
# Special editing
if parms["doEditList"] and not check:
mess = "Special editing"
printMess(mess, logFile)
for edt in parms["editList"]:
UV.PFlag(uv,err,timeRange=[dhms2day(edt["timer"][0]),dhms2day(edt["timer"][1])], \
flagVer=parms["editFG"], Ants=edt["Ant"], Chans=edt["Chans"], IFs=edt["IFs"], \
Stokes=edt["Stokes"], Reason=edt["Reason"])
OErr.printErrMsg(err, "Error Flagging")
# Quack to remove data from start and end of each scan
if parms["doQuack"]:
retCode = EVLAQuack (uv, err, begDrop=parms["quackBegDrop"], endDrop=parms["quackEndDrop"], \
Reason=parms["quackReason"], \
logfile=logFile, check=check, debug=debug)
if retCode != 0:
raise RuntimeError("Error Quacking data")
# Flag antennas shadowed by others?
if parms["doShad"]:
retCode = EVLAShadow (uv, err, shadBl=parms["shadBl"], \
logfile=logFile, check=check, debug=debug)
if retCode != 0:
raise RuntimeError("Error Shadow flagging data")
# Median window time editing, for RFI impulsive in time
if parms["doMedn"]:
mess = "Median window time editing, for RFI impulsive in time:"
printMess(mess, logFile)
retCode = EVLAMedianFlag (uv, " ", err, noScrat=noScrat, nThreads=nThreads, \
avgTime=parms["avgTime"], avgFreq=parms["avgFreq"], chAvg= parms["chAvg"], \
timeWind=parms["timeWind"], flagVer=2,flagSig=parms["mednSigma"], \
logfile=logFile, check=check, debug=False)
if retCode != 0:
raise RuntimeError("Error in MednFlag")
# Median window frequency editing, for RFI impulsive in frequency
if parms["doFD1"]:
mess = "Median window frequency editing, for RFI impulsive in frequency:"
printMess(mess, logFile)
retCode = EVLAAutoFlag (uv, " ", err, flagVer=2, doCalib=-1, doBand=-1, \
timeAvg=parms["FD1TimeAvg"], \
doFD=True, FDmaxAmp=1.0e20, FDmaxV=1.0e20, FDwidMW=parms["FD1widMW"], \
FDmaxRMS=[1.0e20,0.1], FDmaxRes=parms["FD1maxRes"], \
FDmaxResBL= parms["FD1maxRes"], \
nThreads=nThreads, logfile=logFile, check=check, debug=debug)
if retCode != 0:
raise RuntimeError("Error in AutoFlag")
# RMS/Mean editing for calibrators
if parms["doRMSAvg"]:
mess = "RMS/Mean editing for calibrators:"
printMess(mess, logFile)
clist = [] # Calibrator list
for s in parms["ACals"]:
if s['Source'] not in clist:
clist.append(s['Source'])
for s in parms["PCals"]:
if s['Source'] not in clist:
clist.append(s['Source'])
for s in parms["DCals"]:
if s['Source'] not in clist:
clist.append(s['Source'])
retCode = EVLAAutoFlag (uv, clist, err, flagVer=2, doCalib=-1, doBand=-1, \
RMSAvg=parms["RMSAvg"], timeAvg=parms["RMSTimeAvg"], \
nThreads=nThreads, logfile=logFile, check=check, debug=debug)
if retCode != 0:
raise RuntimeError("Error in AutoFlag")
# Need to find a reference antenna? See if we have saved it?
if (parms["refAnt"] <= 0):
refAnt = FetchObject(project + "_" + session + "_" + band +
".refAnt.pickle")
if refAnt:
parms["refAnt"] = refAnt
# Use bandpass calibrator and center half of each spectrum
if parms["refAnt"] <= 0:
mess = "Find best reference antenna: run Calib on BP Cal(s) "
printMess(mess, logFile)
parms["refAnt"] = EVLAGetRefAnt(uv, parms["BPCals"], err, flagVer=2, \
solInt=parms["bpsolint1"], nThreads=nThreads, \
logfile=logFile, check=check, debug=debug)
if err.isErr:
raise RuntimeError("Error finding reference antenna")
if parms["refAnts"][0] <= 0:
parms["refAnts"][0] = parms["refAnt"]
mess = "Picked reference antenna " + str(parms["refAnt"])
printMess(mess, logFile)
# Save it
ParmsPicklefile = project + "_" + session + "_" + band + ".Parms.pickle" # Where results saved
SaveObject(parms, ParmsPicklefile, True)
refAntPicklefile = project + "_" + session + "_" + band + ".refAnt.pickle" # Where results saved
SaveObject(parms["refAnt"], refAntPicklefile, True)
# Plot Raw, edited data?
if parms["doRawSpecPlot"] and parms["plotSource"]:
mess = "Raw Spectral plot for: " + parms["plotSource"]
printMess(mess, logFile)
plotFile = "./" + fileRoot + "RawSpec.ps"
retCode = EVLASpectrum(uv, parms["plotSource"], parms["plotTime"], plotFile, parms["refAnt"], err, \
Stokes=["RR","LL"], doband=-1, \
check=check, debug=debug, logfile=logFile )
if retCode != 0:
raise RuntimeError("Error in Plotting spectrum")
EVLAAddOutFile(plotFile, 'project', 'Pipeline log file')
# delay calibration
if parms["doDelayCal"] and parms["DCals"] and not check:
plotFile = "./" + fileRoot + "DelayCal.ps"
retCode = EVLADelayCal(uv, parms["DCals"], err, \
BChan=parms["delayBChan"], EChan=parms["delayEChan"], \
doCalib=2, flagVer=2, doBand=-1, \
solInt=parms["delaySolInt"], smoTime=1.0/60.0, \
refAnts=[parms["refAnt"]], doTwo=parms["doTwo"],
doZeroPhs=parms["delayZeroPhs"], \
doPlot=parms["doSNPlot"], plotFile=plotFile, \
nThreads=nThreads, noScrat=noScrat, \
logfile=logFile, check=check, debug=debug)
if retCode != 0:
raise RuntimeError("Error in delay calibration")
# Plot corrected data?
if parms["doSpecPlot"] and parms["plotSource"]:
plotFile = "./" + fileRoot + "DelaySpec.ps"
retCode = EVLASpectrum(uv, parms["plotSource"], parms["plotTime"], \
plotFile, parms["refAnt"], err, \
Stokes=["RR","LL"], doband=-1, \
check=check, debug=debug, logfile=logFile )
if retCode != 0:
raise RuntimeError("Error in Plotting spectrum")
# Bandpass calibration
if parms["doBPCal"] and parms["BPCals"]:
retCode = EVLABPCal(uv, parms["BPCals"], err, noScrat=noScrat, solInt1=parms["bpsolint1"], \
solInt2=parms["bpsolint2"], solMode=parms["bpsolMode"], \
BChan1=parms["bpBChan1"], EChan1=parms["bpEChan1"], \
BChan2=parms["bpBChan2"], EChan2=parms["bpEChan2"], ChWid2=parms["bpChWid2"], \
doCenter1=parms["bpDoCenter1"], refAnt=parms["refAnt"], \
UVRange=parms["bpUVRange"], doCalib=2, gainUse=0, flagVer=2, doPlot=False, \
nThreads=nThreads, logfile=logFile, check=check, debug=debug)
if retCode != 0:
raise RuntimeError("Error in Bandpass calibration")
# Plot corrected data?
if parms["doSpecPlot"] and parms["plotSource"]:
plotFile = "./" + fileRoot + "BPSpec.ps"
retCode = EVLASpectrum(uv, parms["plotSource"], parms["plotTime"], plotFile, \
parms["refAnt"], err, Stokes=["RR","LL"], doband=1, \
check=check, debug=debug, logfile=logFile )
if retCode != 0:
raise RuntimeError("Error in Plotting spectrum")
# Constrain X/Y gain
if parms["doXYRelGain"] and (parms["XYRelGainCal"] != None):
mess = "Calibrate X/Y relative gain:"
printMess(mess, logFile)
plotFile = "./" + fileRoot + "XYGainCal.ps"
retCode = EVLACalAP (uv, [], parms["XYRelGainCal"], err,
doCalib=2, doBand=1, BPVer=1, flagVer=2, flagFail=False, \
BChan=parms["ampBChan"], EChan=parms["ampEChan"], \
solInt=parms["solInt"], solSmo=1440., ampScalar=parms["ampScalar"], \
doPlot=parms["doSNPlot"], plotFile=plotFile, refAnt=parms["refAnt"], \
nThreads=nThreads, noScrat=noScrat, logfile=logFile, check=check, debug=debug)
if retCode != 0:
raise RuntimeError("Error calibrating X/Y gain")
parms["avgPol"] = True
# Fix X/Y gain
# Amp & phase Calibrate
if parms["doAmpPhaseCal"]:
plotFile = "./" + fileRoot + "APCal.ps"
retCode = EVLACalAP (uv, [], parms["ACals"], err, PCals=parms["PCals"],
doCalib=2, doBand=1, BPVer=1, flagVer=2, avgPol=parms["avgPol"], \
BChan=parms["ampBChan"], EChan=parms["ampEChan"], \
solInt=parms["solInt"], solSmo=parms["solSmo"], ampScalar=parms["ampScalar"], \
doAmpEdit=parms["doAmpEdit"], ampSigma=parms["ampSigma"], \
ampEditFG=parms["ampEditFG"], \
doPlot=parms["doSNPlot"], plotFile=plotFile, refAnt=parms["refAnt"], \
nThreads=nThreads, noScrat=noScrat, logfile=logFile, check=check, debug=debug)
if retCode != 0:
raise RuntimeError("Error calibrating")
# More editing
if parms["doAutoFlag"]:
mess = "Post calibration editing:"
printMess(mess, logFile)
# if going to redo then only calibrators
if parms["doRecal"]:
# Only calibrators
clist = []
for DCal in DCals:
if DCal["Source"] not in clist:
clist.append(DCal["Source"])
for PCal in PCals:
if PCal["Source"] not in clist:
clist.append(DCal["Source"])
for ACal in ACals:
if ACal["Source"] not in clist:
clist.append(ACal["Source"])
else:
clist = []
retCode = EVLAAutoFlag (uv, clist, err, flagVer=2, \
doCalib=2, gainUse=0, doBand=1, BPVer=1, \
IClip=parms["IClip"], minAmp=parms["minAmp"], timeAvg=parms["timeAvg"], \
doFD=parms["doAFFD"], FDmaxAmp=parms["FDmaxAmp"], FDmaxV=parms["FDmaxV"], \
FDwidMW=parms["FDwidMW"], FDmaxRMS=parms["FDmaxRMS"], \
FDmaxRes=parms["FDmaxRes"], FDmaxResBL=parms["FDmaxResBL"], \
FDbaseSel=parms["FDbaseSel"], \
nThreads=nThreads, logfile=logFile, check=check, debug=debug)
if retCode != 0:
raise RuntimeError("Error in AutoFlag")
# Redo the calibration using new flagging?
if parms["doBPCal2"] == None:
parms["doBPCal2"] = parms["doBPCal"]
if parms["doDelayCal2"] == None:
parms["doDelayCal2"] = parms["doDelayCal2"]
if parms["doXYRelGain2"] == None:
parms["doXYRelGain2"] = parms["doXYRelGain"]
if parms["doAmpPhaseCal2"] == None:
parms["doAmpPhaseCal2"] = parms["doAmpPhaseCal"]
if parms["doAutoFlag2"] == None:
parms["doAutoFlagCal2"] = parms["doAutoFlag"]
if parms["doRecal"]:
mess = "Redo calibration:"
printMess(mess, logFile)
EVLAClearCal(uv,
err,
doGain=True,
doFlag=False,
doBP=True,
check=check,
logfile=logFile)
OErr.printErrMsg(err, "Error resetting calibration")
# Delay recalibration
if parms["doDelayCal2"] and parms["DCals"] and not check:
plotFile = "./" + fileRoot + "DelayCal2.ps"
retCode = EVLADelayCal(uv, parms["DCals"], err, \
BChan=parms["delayBChan"], EChan=parms["delayEChan"], \
doCalib=2, flagVer=2, doBand=-1, \
solInt=parms["delaySolInt"], smoTime=1.0/60.0, \
refAnts=[parms["refAnt"]], doTwo=parms["doTwo"], \
doZeroPhs=parms["delayZeroPhs"], \
doPlot=parms["doSNPlot"], plotFile=plotFile, \
nThreads=nThreads, noScrat=noScrat, \
logfile=logFile, check=check, debug=debug)
if retCode != 0:
raise RuntimeError("Error in delay calibration")
# Plot corrected data?
if parms["doSpecPlot"] and parms["plotSource"]:
plotFile = "./" + fileRoot + "DelaySpec2.ps"
retCode = EVLASpectrum(uv, parms["plotSource"], parms["plotTime"], plotFile, parms["refAnt"], err, \
Stokes=["RR","LL"], doband=-1, \
check=check, debug=debug, logfile=logFile )
if retCode != 0:
raise RuntimeError("Error in Plotting spectrum")
# Bandpass calibration
if parms["doBPCal2"] and parms["BPCals"]:
retCode = EVLABPCal(uv, parms["BPCals"], err, noScrat=noScrat, solInt1=parms["bpsolint1"], \
solInt2=parms["bpsolint2"], solMode=parms["bpsolMode"], \
BChan1=parms["bpBChan1"], EChan1=parms["bpEChan1"], \
BChan2=parms["bpBChan2"], EChan2=parms["bpEChan2"], ChWid2=parms["bpChWid2"], \
doCenter1=parms["bpDoCenter1"], refAnt=parms["refAnt"], \
UVRange=parms["bpUVRange"], doCalib=2, gainUse=0, flagVer=2, doPlot=False, \
nThreads=nThreads, logfile=logFile, check=check, debug=debug)
if retCode != 0:
raise RuntimeError("Error in Bandpass calibration")
# Plot corrected data?
if parms["doSpecPlot"] and parms["plotSource"]:
plotFile = "./" + fileRoot + "BPSpec2.ps"
retCode = EVLASpectrum(uv, parms["plotSource"], parms["plotTime"], plotFile, parms["refAnt"], err, \
Stokes=["RR","LL"], doband=1, \
check=check, debug=debug, logfile=logFile )
if retCode != 0:
raise RuntimeError("Error in Plotting spectrum")
# Constrain X/Y gain
parms["avgPol"] = False
# Fix X/Y gain?
if parms["doXYRelGain2"] and (parms["XYRelGainCal"] != None):
mess = "Calibrate X/Y relative gain:"
printMess(mess, logFile)
plotFile = "./" + fileRoot + "XYGainCal2.ps"
retCode = EVLACalAP (uv, [], parms["XYRelGainCal"], err,
doCalib=2, doBand=1, BPVer=1, flagVer=2, timeRange=parms["XYRelGainCalTime"], \
BChan=parms["ampBChan"], EChan=parms["ampEChan"], flagFail=False, \
solInt=parms["solInt"], solSmo=1440., ampScalar=parms["ampScalar"], \
doPlot=parms["doSNPlot"], plotFile=plotFile, refAnt=parms["refAnt"], \
nThreads=nThreads, noScrat=noScrat, logfile=logFile, check=check, debug=debug)
if retCode != 0:
raise RuntimeError("Error calibrating X/Y gain")
parms["avgPol"] = True
# Fix X/Y gain
# Amp & phase Recalibrate
if parms["doAmpPhaseCal2"]:
plotFile = "./" + fileRoot + "APCal2.ps"
retCode = EVLACalAP (uv, [], parms["ACals"], err, PCals=parms["PCals"], \
doCalib=2, doBand=1, BPVer=1, flagVer=2,avgPol=parms["avgPol"], \
BChan=parms["ampBChan"], EChan=parms["ampEChan"], \
solInt=parms["solInt"], solSmo=parms["solSmo"], ampScalar=parms["ampScalar"], \
doAmpEdit=parms["doAmpEdit"], ampSigma=parms["ampSigma"], \
ampEditFG=parms["ampEditFG"], \
doPlot=parms["doSNPlot"], plotFile=plotFile, refAnt=parms["refAnt"], \
noScrat=noScrat, nThreads=nThreads, logfile=logFile, check=check, debug=debug)
if retCode != 0:
raise RuntimeError("Error calibrating")
# More editing
if parms["doAutoFlag2"]:
mess = "Post recalibration editing:"
printMess(mess, logFile)
retCode = EVLAAutoFlag (uv, [], err, flagVer=2, \
doCalib=2, gainUse=0, doBand=1, BPVer=1, \
IClip=parms["IClip"], minAmp=parms["minAmp"], timeAvg=parms["timeAvg"], \
doFD=parms["doAFFD"], FDmaxAmp=parms["FDmaxAmp"], FDmaxV=parms["FDmaxV"], \
FDwidMW=parms["FDwidMW"], FDmaxRMS=parms["FDmaxRMS"], \
FDmaxRes=parms["FDmaxRes"], FDmaxResBL= parms["FDmaxResBL"], \
FDbaseSel=parms["FDbaseSel"], \
nThreads=nThreads, logfile=logFile, check=check, debug=debug)
if retCode != 0:
raise RuntimeError("Error in AutoFlag")
# end recal
# Calibrate and average data
if parms["doCalAvg"]:
retCode = EVLACalAvg (uv, avgClass, parms["seq"], parms["CalAvgTime"], err, \
flagVer=2, doCalib=2, gainUse=0, doBand=1, BPVer=1, doPol=False, \
avgFreq=parms["avgFreq"], chAvg=parms["chAvg"], \
BChan=parms["CABChan"], EChan=parms["CAEChan"], \
BIF=parms["CABIF"], EIF=parms["CAEIF"], Compress=parms["Compress"], \
nThreads=nThreads, logfile=logFile, check=check, debug=debug)
if retCode != 0:
raise RuntimeError("Error in CalAvg")
# Get calibrated/averaged data
if not check:
uv = UV.newPAUV("AIPS UV DATA", EVLAAIPSName(project, session), avgClass[0:6], \
disk, parms["seq"], True, err)
if err.isErr:
OErr.printErrMsg(err, "Error creating cal/avg AIPS data")
# XClip
if parms["XClip"] and parms["XClip"] > 0.0:
mess = "Cross Pol clipping:"
printMess(mess, logFile)
retCode = EVLAAutoFlag (uv, [], err, flagVer=-1, flagTab=1, \
doCalib=2, gainUse=0, doBand=-1, maxBad=1.0, \
XClip=parms["XClip"], timeAvg=1./60., \
nThreads=nThreads, logfile=logFile, check=check, debug=debug)
if retCode != 0:
raise RuntimeError("Error in AutoFlag")
# R-L delay calibration cal if needed,
if parms["doRLDelay"] and parms["RLDCal"][0][0] != None:
if parms["rlrefAnt"] <= 0:
parms["rlrefAnt"] = parms["refAnt"]
# parms["rlDoBand"] if before average, BPVer=parms["rlBPVer"],
retCode = EVLARLDelay(uv, err,\
RLDCal=parms["RLDCal"], BChan=parms["rlBChan"], \
EChan=parms["rlEChan"], UVRange=parms["rlUVRange"], \
soucode=parms["rlCalCode"], doCalib=parms["rlDoCal"], gainUse=parms["rlgainUse"], \
timerange=parms["rltimerange"], \
# NOT HERE doBand=parms["rlDoBand"], BPVer=parms["rlBPVer"], \
flagVer=parms["rlflagVer"], \
refAnt=parms["rlrefAnt"], doPol=False, \
nThreads=nThreads, noScrat=noScrat, logfile=logFile, \
check=check, debug=debug)
if retCode != 0:
raise RuntimeError("Error in R-L delay calibration")
# Polarization calibration
if parms["doPolCal"]:
if parms["PCRefAnt"] <= 0:
parms["PCRefAnt"] = parms["refAnt"]
retCode = EVLAPolCal(uv, parms["PCInsCals"], err, \
doCalib=2, gainUse=0, doBand=-1, flagVer=0, \
fixPoln=parms["PCFixPoln"], pmodel=parms["PCpmodel"], avgIF=parms["PCAvgIF"], \
solInt=parms["PCSolInt"], refAnt=parms["PCRefAnt"], solType=parms["PCSolType"], \
ChInc=parms["PCChInc"], ChWid=parms["PCChWid"], \
nThreads=nThreads, check=check, debug=debug, noScrat=noScrat, logfile=logFile)
if retCode != 0 and (not check):
raise RuntimeError("Error in polarization calibration: " +
str(retCode))
# end poln cal.
# R-L phase calibration cal., creates new BP table
if parms["doRLCal"] and parms["RLDCal"][0][0] != None:
plotFile = "./" + fileRoot + "RLSpec2.ps"
if parms["rlrefAnt"] <= 0:
parms["rlrefAnt"] = parms["refAnt"]
retCode = EVLARLCal(uv, err,\
RLDCal=parms["RLDCal"], BChan=parms["rlBChan"],
EChan=parms["rlEChan"], UVRange=parms["rlUVRange"], \
ChWid2=parms["rlChWid"], solInt1=parms["rlsolint1"], solInt2=parms["rlsolint2"], \
RLPCal=parms["RLPCal"], RLPhase=parms["RLPhase"], \
RM=parms["RLRM"], CleanRad=parms["rlCleanRad"], \
calcode=parms["rlCalCode"], doCalib=parms["rlDoCal"], gainUse=parms["rlgainUse"], \
timerange=parms["rltimerange"], FOV=parms["rlFOV"], \
doBand=-1, BPVer=1, flagVer=parms["rlflagVer"], \
refAnt=parms["rlrefAnt"], doPol=parms["doPol"], PDVer=parms["PDVer"], \
doPlot=parms["doSpecPlot"], plotFile=plotFile, \
nThreads=nThreads, noScrat=noScrat, logfile=logFile, \
check=check, debug=debug)
if retCode != 0:
raise RuntimeError("Error in RL phase spectrum calibration")
# VClip
if parms["VClip"] and parms["VClip"] > 0.0:
mess = "VPol clipping:"
printMess(mess, logFile)
retCode = EVLAAutoFlag (uv, [], err, flagVer=-1, flagTab=1, \
doCalib=2, gainUse=0, doBand=-1, \
VClip=parms["VClip"], timeAvg=parms["timeAvg"], \
nThreads=nThreads, logfile=logFile, check=check, debug=debug)
if retCode != 0:
raise RuntimeError("Error in AutoFlag VClip")
# Plot corrected data?
if parms["doSpecPlot"] and parms["plotSource"]:
plotFile = "./" + fileRoot + "Spec.ps"
retCode = EVLASpectrum(uv, parms["plotSource"], parms["plotTime"], \
plotFile, parms["refAnt"], err, \
Stokes=["RR","LL"], doband=-1, \
check=check, debug=debug, logfile=logFile )
if retCode != 0:
raise RuntimeError("Error in Plotting spectrum")
# Image targets
if parms["doImage"]:
# If targets not specified, image all
if len(parms["targets"]) <= 0:
slist = EVLAAllSource(uv,
err,
logfile=logFile,
check=check,
debug=debug)
else:
slist = parms["targets"]
EVLAImageTargets (uv, err, Sources=slist, seq=parms["seq"], sclass=outIClass, \
doCalib=2, doBand=1, flagVer=1, doPol=parms["doPol"], PDVer=parms["PDVer"], \
Stokes=parms["Stokes"], FOV=parms["FOV"], Robust=parms["Robust"], Niter=parms["Niter"], \
CleanRad=parms["CleanRad"], minFlux=parms["minFlux"], \
maxPSCLoop=parms["maxPSCLoop"], minFluxPSC=parms["minFluxPSC"], \
solPInt=parms["solPInt"], solPMode=parms["solPMode"], solPType=parms["solPType"], \
maxASCLoop=parms["maxASCLoop"], minFluxASC=parms["minFluxASC"], \
solAInt=parms["solAInt"], solAMode=parms["solAMode"], solAType=parms["solAType"], \
avgPol=parms["avgPol"], avgIF=parms["avgIF"], minSNR = 4.0, refAnt=parms["refAnt"], \
do3D=parms["do3D"], BLFact=parms["BLFact"], BLchAvg=parms["BLchAvg"], \
doMB=parms["doMB"], norder=parms["MBnorder"], maxFBW=parms["MBmaxFBW"], \
PBCor=parms["PBCor"],antSize=parms["antSize"], Beam=parms["Beam"], \
nTaper=parms["nTaper"], Tapers=parms["Tapers"], \
nThreads=nThreads, noScrat=noScrat, logfile=logFile, check=check, debug=debug)
# End image
# Get report on sources
if parms["doReport"]:
# If targets not specified, do all
if len(parms["targets"]) <= 0:
slist = EVLAAllSource(uv,
err,
logfile=logFile,
check=check,
debug=debug)
else:
slist = parms["targets"]
Report = EVLAReportTargets(uv, err, Sources=slist, seq=parms["seq"], sclass=outIClass, \
Stokes=parms["Stokes"], logfile=logFile, check=check, debug=debug)
# Save to pickle jar
ReportPicklefile = "./" + fileRoot + "Report.pickle" # Where results saved
SaveObject(Report, ReportPicklefile, True)
# Write results, cleanup
# Save cal/average UV data?
if parms["doSaveUV"] and (not check):
Aname = EVLAAIPSName(project, session)
cno = AIPSDir.PTestCNO(disk, user, Aname, avgClass[0:6], "UV",
parms["seq"], err)
if cno > 0:
uvt = UV.newPAUV("AIPS CAL UV DATA", Aname, avgClass, disk,
parms["seq"], True, err)
filename = parms["project"] + parms["session"] + parms[
"band"] + "Cal.uvtab"
fuv = EVLAUVFITS(uvt,
filename,
0,
err,
compress=parms["Compress"],
logfile=logFile)
EVLAAddOutFile(filename, 'project', "Calibrated Averaged UV data")
# Save list of output files
EVLASaveOutFiles()
del uvt
# Save raw UV data tables?
if parms["doSaveTab"] and (not check):
Aname = EVLAAIPSName(project, session)
cno = AIPSDir.PTestCNO(disk, user, Aname, dataClass[0:6], "UV",
parms["seq"], err)
if cno > 0:
uvt = UV.newPAUV("AIPS RAW UV DATA", Aname, dataClass[0:6], disk,
parms["seq"], True, err)
filename = parms["project"] + parms["session"] + parms[
"band"] + "CalTab.uvtab"
fuv = EVLAUVFITSTab(uvt, filename, 0, err, logfile=logFile)
EVLAAddOutFile(filename, 'project', "Calibrated AIPS tables")
del uvt
# Write History
filename = project + '_' + session + '_' + band + ".History.text"
OTObit.PrintHistory(uv, file=filename)
EVLAAddOutFile(filename, 'project',
"Processing history of calibrated data")
# Save list of output files
EVLASaveOutFiles()
# Imaging results
# If targets not specified, save all
if len(parms["targets"]) <= 0:
slist = EVLAAllSource(uv,
err,
logfile=logFile,
check=check,
debug=debug)
else:
slist = parms["targets"]
for target in slist:
if parms["doSaveImg"] and (not check):
for s in parms["Stokes"]:
oclass = s + outIClass[1:]
outname = target
# Test if image exists
cno = AIPSDir.PTestCNO(disk, user, outname, oclass, "MA",
parms["seq"], err)
if cno <= 0:
continue
x = Image.newPAImage("out", outname, oclass, disk,
parms["seq"], True, err)
outfile = "./" + fileRoot + target + "." + oclass + ".fits"
xf = EVLAImFITS(x, outfile, 0, err, logfile=logFile)
EVLAAddOutFile(outfile, target, 'Image of ' + target)
# Statistics
zz = imstat(x, err, logfile=logFile)
# end writing loop
# Save list of output files
EVLASaveOutFiles()
OErr.printErrMsg(err, "Writing output")
# Contour plots
if parms["doKntrPlots"]:
mess = "INFO --> Contour plots (doKntrPlots)"
printMess(mess, logFile)
EVLAKntrPlots(err,
imName=parms["targets"],
project=project,
session=session,
band=band,
disk=disk,
debug=debug)
# Save list of output files
EVLASaveOutFiles()
elif debug:
mess = "Not creating contour plots ( doKntrPlots = " + str(
parms["doKntrPlots"]) + " )"
printMess(mess, logFile)
# Source uv plane diagnostic plots
if parms["doDiagPlots"]:
mess = "INFO --> Diagnostic plots (doDiagPlots)"
printMess(mess, logFile)
# Get the highest number avgClass catalog file
Aname = EVLAAIPSName(project, session)
uvc = None
if not check:
uvname = project + "_" + session + "_" + band + "_Cal"
uvc = UV.newPAUV(uvname, Aname, avgClass, disk, parms["seq"], True,
err)
EVLADiagPlots( uvc, err, cleanUp=parms["doCleanup"], \
project=project, session=session, band=band, \
logfile=logFile, check=check, debug=debug )
# Save list of output files
EVLASaveOutFiles()
elif debug:
mess = "Not creating diagnostic plots ( doDiagPlots = " + str(
parms["doDiagPlots"]) + " )"
printMess(mess, logFile)
# Save metadata
srcMetadata = None
projMetadata = None
if parms["doMetadata"]:
mess = "INFO --> Save metadata (doMetadata)"
printMess(mess, logFile)
uvc = None
if not uvc:
# Get calibrated/averaged data
Aname = EVLAAIPSName(project, session)
uvname = project + "_" + session + "_" + band + "_Cal"
uvc = UV.newPAUV(uvname, Aname, avgClass, disk, parms["seq"], True,
err)
if err.isErr:
OErr.printErrMsg(err, "Error creating cal/avg AIPS data")
# Get source metadata; save to pickle file
srcMetadata = EVLASrcMetadata( uvc, err, Sources=parms["targets"], seq=parms["seq"], \
sclass=outIClass, Stokes=parms["Stokes"],\
logfile=logFile, check=check, debug=debug )
picklefile = "./" + fileRoot + ".SrcReport.pickle"
SaveObject(srcMetadata, picklefile, True)
EVLAAddOutFile(picklefile, 'project', 'All source metadata')
# Get project metadata; save to pickle file
projMetadata = EVLAProjMetadata( uvc, AIPS_VERSION, err, \
PCals=parms["PCals"], ACals=parms["ACals"], \
BPCals=parms["BPCals"], DCals=parms["DCals"], \
project = project, session = session, band = band, \
dataInUVF = parms["archRoot"], archFileID = 66666 )
picklefile = "./" + fileRoot + ".ProjReport.pickle"
SaveObject(projMetadata, picklefile, True)
EVLAAddOutFile(picklefile, 'project', 'Project metadata')
else:
# Fetch from pickle jar
picklefile = "./" + fileRoot + ".SrcReport.pickle"
srcMetadata = FetchObject(picklefile)
picklefile = "./" + fileRoot + ".ProjReport.pickle"
projMetadata = FetchObject(picklefile)
# Write report
if parms["doHTML"]:
mess = "INFO --> Write HTML report (doHTML)"
printMess(mess, logFile)
EVLAHTMLReport( projMetadata, srcMetadata, \
outfile="./"+fileRoot+".report.html", \
logFile=logFile )
# Write VOTable
if parms["doVOTable"]:
mess = "INFO --> Write VOTable (doVOTable)"
printMess(mess, logFile)
EVLAAddOutFile('VOTable.xml', 'project', 'VOTable report')
EVLAWriteVOTable(projMetadata, srcMetadata, filename='VOTable.xml')
# Save list of output files
EVLASaveOutFiles()
# Cleanup - delete AIPS files
if parms["doCleanup"] and (not check):
mess = "INFO --> Clean up (doCleanup)"
printMess(mess, logFile)
# Delete target images
# How many Stokes images
nstok = len(parms["Stokes"])
for istok in range(0, nstok):
oclass = parms["Stokes"][istok:istok + 1] + outIClass[1:]
AllDest(err, disk=disk, Aseq=parms["seq"], Aclass=oclass)
# Delete initial UV data
Aname = EVLAAIPSName(project, session)
# Test if data exists
cno = AIPSDir.PTestCNO(disk, user, Aname, dataClass[0:6], "UV",
parms["seq"], err)
if cno > 0:
uvt = UV.newPAUV("AIPS RAW UV DATA", Aname, dataClass[0:6], disk,
parms["seq"], True, err)
uvt.Zap(err)
del uvt
if err.isErr:
OErr.printErrMsg(err, "Error deleting raw AIPS data")
# Zap calibrated/averaged data
# Test if data exists
cno = AIPSDir.PTestCNO(disk, user, Aname, avgClass[0:6], "UV",
parms["seq"], err)
if cno > 0:
uvt = UV.newPAUV("AIPS CAL UV DATA", Aname, avgClass[0:6], disk,
parms["seq"], True, err)
uvt.Zap(err)
del uvt
if err.isErr:
OErr.printErrMsg(err, "Error deleting cal/avg AIPS data")
# Zap UnHanned data if present
loadClass = parms["band"] + "Raw"
# Test if image exists
cno = AIPSDir.PTestCNO(disk, user, Aname, loadClass[0:6], "UV",
parms["seq"], err)
if cno > 0:
uvt = UV.newPAUV("AIPS CAL UV DATA", Aname, loadClass[0:6], disk,
parms["seq"], True, err)
uvt.Zap(err)
del uvt
if err.isErr:
OErr.printErrMsg(err, "Error deleting cal/avg AIPS data")
OErr.printErrMsg(err, "Writing output/cleanup")
# Shutdown
mess = "Finished project "+parms["project"]+" session "+parms["session"]+ \
" "+parms["band"]+" Band"+" AIPS user no. "+str(AIPS.userno)
printMess(mess, logFile)
OErr.printErr(err)
OSystem.Shutdown(ObitSys)
def MKContPipeline(files, outputdir, **kwargs):
"""MeerKAT Continuum pipeline.
Parameters
----------
files : list
h5 filenames (note: support for multiple h5 files
i.e. ConcatenatedDataSet is not currently supported)
outputdir : string
Directory location to write output data,
scratchdir : string, optional
The directory location of the aips disk
parmFile : string, optional
Overwrite the default imaging parameters using this parameter file.
"""
#if len(files) > 1:
# raise TooManyKatfilesException('Processing multiple katfiles are not currently supported')
# Onle be concatenated if we have to be
if len(files) == 1:
h5file = files[0]
else:
h5file = files
# Die gracefully if we cannot write to the output area...
if not os.path.exists(outputdir):
print('Specified output directory: ' + outputdir + 'does not exist.')
exit(-1)
# Obit error logging
err = OErr.OErr()
#################### Initialize filenames #######################################################
fileRoot = os.path.join(outputdir,
os.path.basename(os.path.splitext(
files[0])[0])) # root of file name
logFile = fileRoot + ".log" # Processing log file
avgClass = ("UVAv")[0:6] # Averaged data AIPS class
manifestfile = outputdir + '/manifest.pickle'
############################# Initialize OBIT and AIPS ##########################################
noScrat = []
# Logging directly to logFile
OErr.PInit(err, 2, logFile)
EVLAAddOutFile(os.path.basename(logFile), 'project', 'Pipeline log file')
if kwargs.get('reuse'):
ObitSys = AIPSSetup.AIPSSetup(err,
configfile=kwargs.get('configFile'),
scratchdir=kwargs.get('scratchdir'),
aipsdisk=kwargs.get('aipsdisk'),
overwrite=False)
else:
ObitSys = AIPSSetup.AIPSSetup(err,
configfile=kwargs.get('configFile'),
scratchdir=kwargs.get('scratchdir'),
aipsdisk=kwargs.get('aipsdisk'))
# Get the set up AIPS environment.
AIPS_ROOT = os.environ['AIPS_ROOT']
AIPS_VERSION = os.environ['AIPS_VERSION']
nThreads = 72
user = OSystem.PGetAIPSuser()
AIPS.userno = user
disk = 1
fitsdisk = 0
nam = os.path.basename(os.path.splitext(files[0])[0])[0:10]
cls = "Raw"
seq = 1
############################# Initialise Parameters ##########################################
####### Initialize parameters dictionary #####
parms = KATInitContParms()
####### User defined parameters ######
if kwargs.get('parmFile'):
print("parmFile", kwargs.get('parmFile'))
exec(open(kwargs.get('parmFile')).read())
EVLAAddOutFile(os.path.basename(kwargs.get('parmFile')), 'project',
'Pipeline input parameters')
############### Initialize katfile object, uvfits object and condition data #########################
OK = False
# Open the h5 file as a katfile object
try:
#open katfile and perform selection according to kwargs
katdata = katfile.open(h5file)
OK = True
except Exception as exception:
print(exception)
if not OK:
OErr.PSet(err)
OErr.PLog(err, OErr.Fatal,
"Unable to read KAT HDF5 data in " + str(h5file))
raise KATUnimageableError("Unable to read KAT HDF5 data in " +
str(h5file))
#Are we MeerKAT or KAT-7
telescope = katdata.ants[0].name[0]
if telescope == 'm':
sefd = 500.
else:
sefd = 1200.
#Get calibrator models
fluxcals = katpoint.Catalogue(
open(FITSDir.FITSdisks[0] + "/" + parms["fluxModel"]))
#Condition data (get bpcals, update names for aips conventions etc)
KATh5Condition(katdata, fluxcals, err)
###################### Data selection and static edits ############################################
# Select data based on static imageable parameters
MKATh5Select(katdata, parms, err, **kwargs)
# General AIPS data parameters at script level
dataClass = ("UVDa")[0:6] # AIPS class of raw uv data
band = katdata.spectral_windows[0].product #Correlator product
project = os.path.basename(os.path.splitext(files[0])[0])[
0:10] # Project name (12 char or less, used as AIPS Name)
outIClass = parms["outIClass"] # image AIPS class
debug = parms["debug"]
check = parms["check"]
####################### Import data into AIPS #####################################################
# Reuse or nay?
if kwargs.get('reuse'):
uv = UV.newPAUV("AIPS UV DATA", EVLAAIPSName(project), dataClass, disk,
seq, True, err)
obsdata = KATH5toAIPS.GetKATMeta(katdata, err)
# Extract AIPS parameters of the uv data to the metadata
obsdata["Aproject"] = uv.Aname
obsdata["Aclass"] = uv.Aclass
obsdata["Aseq"] = uv.Aseq
obsdata["Adisk"] = disk
obsdata["calInt"] = katdata.dump_period
obsdata["fitsdisk"] = fitsdisk
# TODO: Check if the input data has been Hanned.
doneHann = True
else:
# Number of baselines gives batch size
nbl = len(
np.unique([(cp[0][:-1] + cp[1][:-1]).upper()
for cp in katdata.corr_products]))
# Construct a template uvfits file from master template
mastertemplate = ObitTalkUtil.FITSDir.FITSdisks[
fitsdisk] + 'MKATTemplate.uvtab.gz'
outtemplate = nam + '.uvtemp'
KATH5toAIPS.MakeTemplate(mastertemplate,
outtemplate,
len(katdata.channel_freqs),
nvispio=nbl)
uv = OTObit.uvlod(outtemplate, 0, EVLAAIPSName(project), cls, disk,
seq, err)
obsdata = KATH5toAIPS.KAT2AIPS(katdata,
uv,
disk,
fitsdisk,
err,
calInt=katdata.dump_period,
**kwargs)
MakeIFs.UVMakeIF(uv, 8, err)
os.remove(outtemplate)
# Print the uv data header to screen.
uv.Header(err)
############################# Set Project Processing parameters ###################################
# Parameters derived from obsdata and katdata
MKATGetObsParms(obsdata, katdata, parms, logFile)
###### Initialise target parameters #####
KATInitTargParms(katdata, parms, err)
# Load the outputs pickle jar
EVLAFetchOutFiles()
OSystem.PAllowThreads(nThreads) # Allow threads in Obit/oython
retCode = 0
maxgap = max(parms["CalAvgTime"], 20 * katdata.dump_period) / 60.
################### Start processing ###############################################################
mess = "Start project "+parms["project"]+" AIPS user no. "+str(AIPS.userno)+\
", KAT7 configuration "+parms["KAT7Cfg"]
printMess(mess, logFile)
if debug:
pydoc.ttypager = pydoc.plainpager # don't page task input displays
mess = "Using Debug mode "
printMess(mess, logFile)
if check:
mess = "Only checking script"
printMess(mess, logFile)
# Log parameters
printMess("Parameter settings", logFile)
for p in parms:
mess = " " + p + ": " + str(parms[p])
printMess(mess, logFile)
clist = []
for DCal in parms["DCals"]:
if DCal["Source"] not in clist:
clist.append(DCal["Source"])
for PCal in parms["PCals"]:
if PCal["Source"] not in clist:
clist.append(PCal["Source"])
for ACal in parms["ACals"]:
if ACal["Source"] not in clist:
clist.append(ACal["Source"])
if kwargs.get('targets') is not None:
targets = [
targ.name for targ in katdata.catalogue
if (targ.name not in clist) and (
targ.name in kwargs.get('targets').split(','))
]
else:
targets = [
targ.name for targ in katdata.catalogue if (targ.name not in clist)
]
refAnt = FetchObject(fileRoot + ".refAnt.pickle")
# Save parameters to pickle jar, manifest
ParmsPicklefile = fileRoot + ".Parms.pickle" # Where results saved
SaveObject(parms, ParmsPicklefile, True)
EVLAAddOutFile(os.path.basename(ParmsPicklefile), 'project',
'Processing parameters used')
loadClass = dataClass
# Hanning - No Hanning
parms["doHann"] = False
doneHann = False
if doneHann:
# Halve channels after hanning.
parms["selChan"] = int(parms["selChan"] / 2)
parms["BChDrop"] = int(parms["BChDrop"] / 2)
parms["EChDrop"] = int(parms["EChDrop"] / 2)
if uv == None and not check:
raise RuntimeError("Cannot Hann data ")
# Clear any old calibration/editing
if parms["doClearTab"] or kwargs.get('reuse'):
mess = "Clear previous calibration"
printMess(mess, logFile)
EVLAClearCal(uv,
err,
doGain=parms["doClearGain"],
doFlag=parms["doClearFlag"],
doBP=parms["doClearBP"],
check=check)
OErr.printErrMsg(err, "Error resetting calibration")
# Quack to remove data from start and end of each scan
if parms["doQuack"]:
retCode = EVLAQuack (uv, err, begDrop=parms["quackBegDrop"], endDrop=parms["quackEndDrop"], \
Reason=parms["quackReason"], \
logfile=logFile, check=check, debug=debug)
if retCode != 0:
raise RuntimeError("Error Quacking data")
# Flag antennas shadowed by others?
if parms["doShad"]:
retCode = EVLAShadow (uv, err, shadBl=parms["shadBl"], \
logfile=logFile, check=check, debug=debug)
if retCode != 0:
raise RuntimeError("Error Shadow flagging data")
# Median window time editing, for RFI impulsive in time
if parms["doMednTD1"]:
mess = "Median window time editing, for RFI impulsive in time:"
printMess(mess, logFile)
retCode = EVLAMedianFlag (uv, clist, err, noScrat=noScrat, nThreads=nThreads, \
avgTime=parms["mednAvgTime"], avgFreq=parms["mednAvgFreq"], chAvg= parms["mednChAvg"], \
timeWind=parms["mednTimeWind"],flagVer=2, flagTab=2,flagSig=parms["mednSigma"], \
logfile=logFile, check=check, debug=False)
if retCode != 0:
raise RuntimeError("Error in MednFlag")
# Median window frequency editing, for RFI impulsive in frequency
if parms["doFD1"]:
mess = "Median window frequency editing, for RFI impulsive in frequency:"
printMess(mess, logFile)
retCode = EVLAAutoFlag (uv, clist, err, flagVer=2, flagTab=2, doCalib=-1, doBand=-1, \
timeAvg=parms["FD1TimeAvg"], \
doFD=True, FDmaxAmp=1.0e20, FDmaxV=1.0e20, FDwidMW=parms["FD1widMW"], \
FDmaxRMS=[1.0e20,0.1], FDmaxRes=parms["FD1maxRes"], \
FDmaxResBL= parms["FD1maxRes"], FDbaseSel=parms["FD1baseSel"],\
nThreads=nThreads, logfile=logFile, check=check, debug=debug)
if retCode != 0:
raise RuntimeError("Error in AutoFlag")
# Parallactic angle correction?
if parms["doPACor"]:
retCode = EVLAPACor(uv, err, \
logfile=logFile, check=check, debug=debug)
if retCode != 0:
raise RuntimeError("Error in Parallactic angle correction")
# Need to find a reference antenna? See if we have saved it?
if (parms["refAnt"] <= 0):
refAnt = FetchObject(fileRoot + ".refAnt.pickle")
if refAnt:
parms["refAnt"] = refAnt
# Use bandpass calibrator and center half of each spectrum
if parms["refAnt"] <= 0:
mess = "Find best reference antenna: run Calib on BP Cal(s) "
printMess(mess, logFile)
parms["refAnt"] = EVLAGetRefAnt(uv, parms["BPCals"], err, flagVer=0, \
solInt=parms["bpsolint1"], nThreads=nThreads, \
logfile=logFile, check=check, debug=debug)
if err.isErr:
raise RuntimeError("Error finding reference antenna")
if parms["refAnts"][0] <= 0:
parms["refAnts"][0] = parms["refAnt"]
mess = "Picked reference antenna " + str(parms["refAnt"])
printMess(mess, logFile)
# Save it
ParmsPicklefile = fileRoot + ".Parms.pickle" # Where results saved
SaveObject(parms, ParmsPicklefile, True)
refAntPicklefile = fileRoot + ".refAnt.pickle" # Where results saved
SaveObject(parms["refAnt"], refAntPicklefile, True)
# Plot Raw, edited data?
parms["doRawSpecPlot"] = False
parms["doSpecPlot"] = False
if parms["doRawSpecPlot"] and parms["plotSource"]:
mess = "Raw Spectral plot for: " + ' '.join(parms["BPCal"])
printMess(mess, logFile)
plotFile = fileRoot + "_RawSpec.ps"
retCode = EVLASpectrum(uv, parms["BPCal"], parms["plotTime"], maxgap, plotFile, parms["refAnt"], err, \
Stokes=["RR","LL"], doband=-1, \
check=check, debug=debug, logfile=logFile )
if retCode != 0:
raise RuntimeError("Error in Plotting spectrum")
EVLAAddOutFile(plotFile, 'project', 'Pipeline log file')
# delay calibration
if parms["doDelayCal"] and parms["DCals"] and not check:
plotFile = fileRoot + "_DelayCal.ps"
retCode = EVLADelayCal(uv, parms["DCals"], err, \
BChan=parms["delayBChan"], EChan=parms["delayEChan"], \
doCalib=2, flagVer=0, doBand=-1, \
solInt=parms["delaySolInt"], smoTime=parms["delaySmoo"], \
refAnts=[parms["refAnt"]], doTwo=parms["doTwo"],
doZeroPhs=parms["delayZeroPhs"], \
doPlot=parms["doSNPlot"], plotFile=plotFile, \
nThreads=nThreads, noScrat=noScrat, \
logfile=logFile, check=check, debug=debug)
if retCode != 0:
raise RuntimeError("Error in delay calibration")
# Plot corrected data?
if parms["doSpecPlot"] and parms["plotSource"]:
plotFile = fileRoot + "_DelaySpec.ps"
retCode = EVLASpectrum(uv, parms["BPCal"], parms["plotTime"], maxgap, \
plotFile, parms["refAnt"], err, \
Stokes=["RR","LL"], doband=-1, \
check=check, debug=debug, logfile=logFile )
if retCode != 0:
raise RuntimeError("Error in Plotting spectrum")
print(parms["bpBChan1"], parms["bpEChan1"], parms["bpBChan2"],
parms["bpEChan2"], parms["bpChWid2"])
# Bandpass calibration
if parms["doBPCal"] and parms["BPCals"]:
retCode = KATBPCal(uv, parms["BPCals"], err, noScrat=noScrat, solInt1=parms["bpsolint1"], \
solInt2=parms["bpsolint2"], solMode=parms["bpsolMode"], \
BChan1=parms["bpBChan1"], EChan1=parms["bpEChan1"], \
BChan2=parms["bpBChan2"], EChan2=parms["bpEChan2"], ChWid2=parms["bpChWid2"], \
doCenter1=parms["bpDoCenter1"], refAnt=parms["refAnt"], \
UVRange=parms["bpUVRange"], doCalib=2, gainUse=0, flagVer=0, doPlot=False, \
nThreads=nThreads, logfile=logFile, check=check, debug=debug)
if retCode != 0:
raise RuntimeError("Error in Bandpass calibration")
# Plot corrected data?
if parms["doSpecPlot"] and parms["plotSource"]:
plotFile = fileRoot + "_BPSpec.ps"
retCode = EVLASpectrum(uv, parms["BPCal"], parms["plotTime"], maxgap, plotFile, \
parms["refAnt"], err, Stokes=["RR","LL"], doband=2, \
check=check, debug=debug, logfile=logFile )
if retCode != 0:
raise RuntimeError("Error in Plotting spectrum")
# Amp & phase Calibrate
if parms["doAmpPhaseCal"]:
plotFile = fileRoot + "_APCal.ps"
retCode = KATCalAP (uv, [], parms["ACals"], err, PCals=parms["PCals"],
doCalib=2, doBand=2, BPVer=1, flagVer=0, \
BChan=parms["ampBChan"], EChan=parms["ampEChan"], \
solInt=parms["solInt"], solSmo=parms["solSmo"], ampScalar=parms["ampScalar"], \
doAmpEdit=parms["doAmpEdit"], ampSigma=parms["ampSigma"], \
ampEditFG=parms["ampEditFG"], \
doPlot=parms["doSNPlot"], plotFile=plotFile, refAnt=parms["refAnt"], \
nThreads=nThreads, noScrat=noScrat, logfile=logFile, check=check, debug=debug)
#print parms["ACals"],parms["PCals"]
if retCode != 0:
raise RuntimeError("Error calibrating")
# More editing
if parms["doAutoFlag"]:
mess = "Post calibration editing:"
printMess(mess, logFile)
# if going to redo then only calibrators
if parms["doRecal"]:
# Only calibrators
clist = []
for DCal in parms["DCals"]:
if DCal["Source"] not in clist:
clist.append(DCal["Source"])
for PCal in parms["PCals"]:
if PCal["Source"] not in clist:
clist.append(PCal["Source"])
for ACal in parms["ACals"]:
if ACal["Source"] not in clist:
clist.append(ACal["Source"])
else:
clist = []
retCode = EVLAAutoFlag (uv, clist, err, flagVer=0, flagTab =2, \
doCalib=2, gainUse=0, doBand=2, BPVer=1, \
IClip=parms["IClip"], minAmp=parms["minAmp"], timeAvg=parms["timeAvg"], \
doFD=parms["doFirstAFFD"], FDmaxAmp=parms["FDmaxAmp"], FDmaxV=parms["FDmaxV"], \
FDwidMW=parms["FDwidMW"], FDmaxRMS=parms["FDmaxRMS"], \
FDmaxRes=parms["FDmaxRes"], FDmaxResBL=parms["FDmaxResBL"], \
FDbaseSel=parms["FDbaseSel"], \
nThreads=nThreads, logfile=logFile, check=check, debug=debug)
if retCode != 0:
raise RuntimeError("Error in AutoFlag")
# Redo the calibration using new flagging?
if parms["doBPCal2"] == None:
parms["doBPCal2"] = parms["doBPCal"]
if parms["doDelayCal2"] == None:
parms["doDelayCal2"] = parms["doDelayCal2"]
if parms["doAmpPhaseCal2"] == None:
parms["doAmpPhaseCal2"] = parms["doAmpPhaseCal"]
if parms["doAutoFlag2"] == None:
parms["doAutoFlagCal2"] = parms["doAutoFlag"]
if parms["doRecal"]:
mess = "Redo calibration:"
printMess(mess, logFile)
EVLAClearCal(uv,
err,
doGain=True,
doFlag=False,
doBP=True,
check=check,
logfile=logFile)
OErr.printErrMsg(err, "Error resetting calibration")
# Parallactic angle correction?
if parms["doPACor"]:
retCode = EVLAPACor(uv, err, \
logfile=logFile, check=check, debug=debug)
if retCode != 0:
raise RuntimeError("Error in Parallactic angle correction")
# Delay recalibration
if parms["doDelayCal2"] and parms["DCals"] and not check:
plotFile = fileRoot + "_DelayCal2.ps"
retCode = EVLADelayCal(uv, parms["DCals"], err, \
BChan=parms["delayBChan"], EChan=parms["delayEChan"], \
doCalib=2, flagVer=0, doBand=-1, \
solInt=parms["delaySolInt"], smoTime=parms["delaySmoo"], \
refAnts=[parms["refAnt"]], doTwo=parms["doTwo"], \
doZeroPhs=parms["delayZeroPhs"], \
doPlot=parms["doSNPlot"], plotFile=plotFile, \
nThreads=nThreads, noScrat=noScrat, \
logfile=logFile, check=check, debug=debug)
if retCode != 0:
raise RuntimeError("Error in delay calibration")
# Plot corrected data?
if parms["doSpecPlot"] and parms["plotSource"]:
plotFile = fileRoot + "_DelaySpec2.ps"
retCode = EVLASpectrum(uv, parms["BPCal"], parms["plotTime"], maxgap, plotFile, parms["refAnt"], err, \
Stokes=["RR","LL"], doband=-1, \
check=check, debug=debug, logfile=logFile )
if retCode != 0:
raise RuntimeError("Error in Plotting spectrum")
# Bandpass calibration
if parms["doBPCal2"] and parms["BPCals"]:
retCode = KATBPCal(uv, parms["BPCals"], err, noScrat=noScrat, solInt1=parms["bpsolint1"], \
solInt2=parms["bpsolint2"], solMode=parms["bpsolMode"], \
BChan1=parms["bpBChan1"], EChan1=parms["bpEChan1"], \
BChan2=parms["bpBChan2"], EChan2=parms["bpEChan2"], ChWid2=parms["bpChWid2"], \
doCenter1=parms["bpDoCenter1"], refAnt=parms["refAnt"], \
UVRange=parms["bpUVRange"], doCalib=2, gainUse=0, flagVer=0, doPlot=False, \
nThreads=nThreads, logfile=logFile, check=check, debug=debug)
if retCode != 0:
raise RuntimeError("Error in Bandpass calibration")
# Plot corrected data?
if parms["doSpecPlot"] and parms["plotSource"]:
plotFile = fileRoot + "_BPSpec2.ps"
retCode = EVLASpectrum(uv, parms["BPCal"], parms["plotTime"], maxgap, plotFile, parms["refAnt"], err, \
Stokes=["RR","LL"], doband=2, \
check=check, debug=debug, logfile=logFile )
if retCode != 0:
raise RuntimeError("Error in Plotting spectrum")
# Amp & phase Recalibrate
if parms["doAmpPhaseCal2"]:
plotFile = fileRoot + "_APCal2.ps"
retCode = KATCalAP (uv, [], parms["ACals"], err, PCals=parms["PCals"], \
doCalib=2, doBand=2, BPVer=1, flagVer=0, \
BChan=parms["ampBChan"], EChan=parms["ampEChan"], \
solInt=parms["solInt"], solSmo=parms["solSmo"], ampScalar=parms["ampScalar"], \
doAmpEdit=True, ampSigma=parms["ampSigma"], \
ampEditFG=parms["ampEditFG"], \
doPlot=parms["doSNPlot"], plotFile=plotFile, refAnt=parms["refAnt"], \
noScrat=noScrat, nThreads=nThreads, logfile=logFile, check=check, debug=debug)
if retCode != 0:
raise RuntimeError("Error calibrating")
# More editing
parms["doAutoFlag2"] = False
if parms["doAutoFlag2"]:
mess = "Post recalibration editing:"
printMess(mess, logFile)
retCode = EVLAAutoFlag (uv, [], err, flagVer=0, flagTab=2, \
doCalib=2, gainUse=0, doBand=2, BPVer=1, \
IClip=parms["IClip"], minAmp=parms["minAmp"], timeAvg=parms["timeAvg"], \
doFD=parms["doSecAFFD"], FDmaxAmp=parms["FDmaxAmp"], FDmaxV=parms["FDmaxV"], \
FDwidMW=parms["FDwidMW"], FDmaxRMS=parms["FDmaxRMS"], \
FDmaxRes=parms["FDmaxRes"], FDmaxResBL= parms["FDmaxResBL"], \
FDbaseSel=parms["FDbaseSel"], \
nThreads=nThreads, logfile=logFile, check=check, debug=debug)
if retCode != 0:
raise RuntimeError("Error in AutoFlag")
# end recal
# Calibrate and average data
# Overwrite avgStokes from command line
parms["avgFreq"] = 0
parms["chAvg"] = 1
parms["doCalAvg"] = 'Splat'
if kwargs.get('halfstokes'):
parms["avgStokes"] = 'HALF'
if parms["doCalAvg"] == 'Splat':
retCode = KATCalAvg (uv, avgClass, parms["seq"], parms["CalAvgTime"], err, \
flagVer=2, doCalib=2, gainUse=0, doBand=2, BPVer=1, doPol=False, \
avgFreq=parms["avgFreq"], chAvg=parms["chAvg"], Stokes=parms["avgStokes"], \
BChan=1, EChan=parms["selChan"] - 1, doAuto=parms["doAuto"], \
BIF=parms["CABIF"], EIF=parms["CAEIF"], Compress=parms["Compress"], \
nThreads=nThreads, logfile=logFile, check=check, debug=debug)
if retCode != 0:
raise RuntimeError("Error in CalAvg")
elif parms["doCalAvg"] == 'BL':
retCode = KATBLCalAvg (uv, avgClass, parms["seq"], err, \
flagVer=2, doCalib=2, gainUse=0, doBand=2, BPVer=1, doPol=False, \
avgFreq=parms["avgFreq"], chAvg=parms["chAvg"], FOV=parms['FOV'], \
maxInt=min(parms["solPInt"],parms["solAInt"]), Stokes=parms["avgStokes"], \
BChan=1, EChan=parms["selChan"] - 1, timeAvg=parms["CalAvgTime"], \
BIF=parms["CABIF"], EIF=parms["CAEIF"], Compress=parms["Compress"], \
logfile=logFile, check=check, debug=debug)
if retCode != 0:
raise RuntimeError("Error in BLCalAvg")
if parms["doSaveTab"]:
filename = project + ".CalTab.uvtab"
_ = EVLAUVFITSTab(uv, filename, 0, err, logfile=logFile)
#Zap unaveraged data if requested
if kwargs.get('zapraw'):
uv.Zap(err)
# Get calibrated/averaged data
if not check:
uv = UV.newPAUV("AIPS UV DATA", EVLAAIPSName(project), avgClass[0:6], \
disk, parms["seq"], True, err)
if err.isErr:
OErr.printErrMsg(err, "Error creating cal/avg AIPS data")
plotFile = fileRoot + "_Spec.ps"
retCode = EVLASpectrum(uv, parms["BPCal"], parms["plotTime"], maxgap, \
plotFile, parms["refAnt"], err, \
Stokes=["I"], doband=-1, docalib=-1, \
check=check, debug=debug, logfile=logFile )
if retCode != 0:
raise RuntimeError("Error in Plotting spectrum")
# KATUVFITS(uv, 'preimage.uvfits', 0, err, exclude=["AIPS HI", "AIPS SL", "AIPS PL"],
# include=["AIPS AN", "AIPS FQ"], compress=parms["Compress"], logfile=logFile)
KATUVFITab(uv, project + '.uvtab', 0, err)
#Gzip the data?
if kwargs.get('gzip'):
os.system('pigz -p %d %s' % (nThreads, project + '.uvtab'))
os.system('rm -f %s' % (project + '.uvtab'))
doSaveUV = True # Save uv data
doSaveImg = True # Save images
doSaveTab = True # Save Tables
doCleanup = True # Destroy AIPS files
# diagnostics
doSNPlot = True # Plot SN tables etc
prtLv = 2 # Amount of task print diagnostics
############################# Set Project Processing parameters ##################
parmFile = sys.argv[2]
execfile(parmFile)
################################## Process #####################################
# Logging directly to logFile
OErr.PInit(err, prtLv, logFile)
retCode = 0
mess = "Start project " + project + " session " + session + " " + band + " Band"
printMess(mess, logFile)
if debug:
mess = "Using Debug mode "
printMess(mess, logFile)
if check:
mess = "Only checking script"
printMess(mess, logFile)
# Load Data from FITS
uv = None
if doLoadIDI:
uv = VLBAIDILoad(dataIn, project, session, band, dataClass, disk, seq, err, logfile=logFile, \
def MKContPipeline(files, outputdir, **kwargs):
"""MeerKAT Continuum pipeline.
Parameters
----------
files : list
h5 filenames (note: support for multiple h5 files
i.e. ConcatenatedDataSet is not currently supported)
outputdir : string
Directory location to write output data,
scratchdir : string, optional
The directory location of the aips disk
parmFile : string, optional
Overwrite the default imaging parameters using this parameter file.
"""
#if len(files) > 1:
# raise TooManyKatfilesException('Processing multiple katfiles are not currently supported')
h5file = files
# Die gracefully if we cannot write to the output area...
if not os.path.exists(outputdir):
print('Specified output directory: '+ outputdir + 'does not exist.')
exit(-1)
# Obit error logging
err = OErr.OErr()
#################### Initialize filenames #######################################################
fileRoot = os.path.join(outputdir, os.path.basename(os.path.splitext(files[0])[0])) # root of file name
logFile = fileRoot+".log" # Processing log file
avgClass = ("UVAv")[0:6] # Averaged data AIPS class
manifestfile = outputdir + '/manifest.pickle'
############################# Initialize OBIT and AIPS ##########################################
noScrat = []
# Logging directly to logFile
OErr.PInit(err, 2, logFile)
EVLAAddOutFile(os.path.basename(logFile), 'project', 'Pipeline log file')
ObitSys = AIPSSetup.AIPSSetup(err,configfile=kwargs.get('configFile'),scratchdir=kwargs.get('scratchdir'))
# Get the set up AIPS environment.
AIPS_ROOT = os.environ['AIPS_ROOT']
AIPS_VERSION = os.environ['AIPS_VERSION']
nThreads = 24
user = OSystem.PGetAIPSuser()
AIPS.userno = user
disk = 1
fitsdisk = 0
nam = os.path.basename(os.path.splitext(files[0])[0])[0:10]
cls = "Raw"
seq = 1
############################# Initialise Parameters ##########################################
####### Initialize parameters dictionary #####
parms = KATInitContParms()
####### User defined parameters ######
if kwargs.get('parmFile'):
print("parmFile",kwargs.get('parmFile'))
exec(open(kwargs.get('parmFile')).read())
EVLAAddOutFile(os.path.basename(kwargs.get('parmFile')), 'project', 'Pipeline input parameters' )
############### Initialize katfile object, uvfits object and condition data #########################
OK = False
# Open the h5 file as a katfile object
try:
#open katfile and perform selection according to kwargs
katdata = katfile.open(h5file)
OK = True
except Exception as exception:
print(exception)
if not OK:
OErr.PSet(err)
OErr.PLog(err, OErr.Fatal, "Unable to read KAT HDF5 data in " + str(h5file))
raise KATUnimageableError("Unable to read KAT HDF5 data in " + str(h5file))
#We have a katdal object- read some flags and ad them in in available
if kwargs.get('flags') is not None:
flags=kwargs.get('flags')
fa = flags.split(',')
for fn,ff in enumerate(fa):
ex_flags_file = h5py.File(ff)
ex_flags = da.from_array(ex_flags_file['flags'], chunks=(1,342,katdata.shape[2]))
#Sum the new flags
katdata.datasets[fn].source.data.flags = ex_flags
#Are we MeerKAT or KAT-7
telescope = katdata.ants[0].name[0]
if telescope=='m':
sefd=500.
else:
sefd=1200.
#Get calibrator models
fluxcals = katpoint.Catalogue(file(FITSDir.FITSdisks[0]+"/"+parms["fluxModel"]))
#Condition data (get bpcals, update names for aips conventions etc)
KATh5Condition(katdata,fluxcals,err)
###################### Data selection and static edits ############################################
# Select data based on static imageable parameters
MKATh5Select(katdata, parms, err, **kwargs)
####################### Import data into AIPS #####################################################
# Construct a template uvfits file from master template
mastertemplate=ObitTalkUtil.FITSDir.FITSdisks[fitsdisk]+'MKATTemplate.uvtab.gz'
outtemplate=nam+'.uvtemp'
KATH5toAIPS.MakeTemplate(mastertemplate,outtemplate,len(katdata.channel_freqs))
uv=OTObit.uvlod(outtemplate,0,nam,cls,disk,seq,err)
obsdata = KATH5toAIPS.KAT2AIPS(katdata, uv, disk, fitsdisk, err, calInt=1.0, **kwargs)
MakeIFs.UVMakeIF(uv,8,err)
# Print the uv data header to screen.
uv.Header(err)
os.remove(outtemplate)
############################# Set Project Processing parameters ###################################
# Parameters derived from obsdata and katdata
MKATGetObsParms(obsdata, katdata, parms, logFile)
###### Initialise target parameters #####
KATInitTargParms(katdata,parms,err)
# General AIPS data parameters at script level
dataClass = ("UVDa")[0:6] # AIPS class of raw uv data
band = katdata.spectral_windows[0].product #Correlator product
project = parms["project"][0:12] # Project name (12 char or less, used as AIPS Name)
outIClass = parms["outIClass"] # image AIPS class
debug = parms["debug"]
check = parms["check"]
# Load the outputs pickle jar
EVLAFetchOutFiles()
OSystem.PAllowThreads(nThreads) # Allow threads in Obit/oython
retCode = 0
maxgap = max(parms["CalAvgTime"],20*katdata.dump_period)/60.
################### Start processing ###############################################################
mess = "Start project "+parms["project"]+" AIPS user no. "+str(AIPS.userno)+\
", KAT7 configuration "+parms["KAT7Cfg"]
printMess(mess, logFile)
if debug:
pydoc.ttypager = pydoc.plainpager # don't page task input displays
mess = "Using Debug mode "
printMess(mess, logFile)
if check:
mess = "Only checking script"
printMess(mess, logFile)
# Log parameters
printMess("Parameter settings", logFile)
for p in parms:
mess = " "+p+": "+str(parms[p])
printMess(mess, logFile)
clist = []
for DCal in parms["DCals"]:
if DCal["Source"] not in clist:
clist.append(DCal["Source"])
for PCal in parms["PCals"]:
if PCal["Source"] not in clist:
clist.append(PCal["Source"])
for ACal in parms["ACals"]:
if ACal["Source"] not in clist:
clist.append(ACal["Source"])
if kwargs.get('targets') is not None:
targets = [targ.name for targ in katdata.catalogue if (targ.name not in clist) and (targ.name in kwargs.get('targets').split(','))]
else:
targets = [targ.name for targ in katdata.catalogue if (targ.name not in clist)]
refAnt = FetchObject(fileRoot+".refAnt.pickle")
# Save parameters to pickle jar, manifest
ParmsPicklefile = fileRoot+".Parms.pickle" # Where results saved
SaveObject(parms, ParmsPicklefile, True)
EVLAAddOutFile(os.path.basename(ParmsPicklefile), 'project', 'Processing parameters used' )
loadClass = dataClass
retCode = KATCalAvg (uv, "PREAVG", parms["seq"], parms["CalAvgTime"], err, \
flagVer=-1, doCalib=-1, gainUse=-1, doBand=-1, BPVer=-1, doPol=False, \
avgFreq=0, chAvg=1, BChan=1, EChan=0, doAuto=parms["doAuto"], Stokes=' ',\
BIF=parms["CABIF"], EIF=parms["CAEIF"], Compress=parms["Compress"], \
nThreads=nThreads, logfile=logFile, check=check, debug=debug)
if retCode!=0:
raise RuntimeError("Error in CalAvg")
uv.Zap(err)
# Get initially averaged data
if not check:
uv = UV.newPAUV("AIPS UV DATA", EVLAAIPSName(project), "PREAVG", \
disk, parms["seq"], True, err)
if err.isErr:
OErr.printErrMsg(err, "Error creating initial avg AIPS data")
# Hanning
parms["doHann"]=True
if parms["doHann"]:
# Set uv if not done
if uv==None and not check:
uv = UV.newPAUV("AIPS UV DATA", EVLAAIPSName(project), loadClass[0:6], disk, parms["seq"], True, err)
if err.isErr:
OErr.printErrMsg(err, "Error creating AIPS data")
uv = KATHann(uv, EVLAAIPSName(project), dataClass, disk, parms["seq"], err, \
doDescm=parms["doDescm"], flagVer=0, logfile=logFile, check=check, debug=debug)
#Halve channels after hanning.
parms["selChan"]=int(parms["selChan"]/2)
parms["BChDrop"]=int(parms["BChDrop"]/2)
parms["EChDrop"]=int(parms["EChDrop"]/2)
if uv==None and not check:
raise RuntimeError("Cannot Hann data ")
# Clear any old calibration/editing
if parms["doClearTab"]:
mess = "Clear previous calibration"
printMess(mess, logFile)
EVLAClearCal(uv, err, doGain=parms["doClearGain"], doFlag=parms["doClearFlag"], doBP=parms["doClearBP"], check=check)
OErr.printErrMsg(err, "Error resetting calibration")
# Quack to remove data from start and end of each scan
if parms["doQuack"]:
retCode = EVLAQuack (uv, err, begDrop=parms["quackBegDrop"], endDrop=parms["quackEndDrop"], \
Reason=parms["quackReason"], \
logfile=logFile, check=check, debug=debug)
if retCode!=0:
raise RuntimeError("Error Quacking data")
# Flag antennas shadowed by others?
if parms["doShad"]:
retCode = EVLAShadow (uv, err, shadBl=parms["shadBl"], \
logfile=logFile, check=check, debug=debug)
if retCode!=0:
raise RuntimeError("Error Shadow flagging data")
# Median window time editing, for RFI impulsive in time
if parms["doMednTD1"]:
mess = "Median window time editing, for RFI impulsive in time:"
printMess(mess, logFile)
retCode = EVLAMedianFlag (uv, clist, err, noScrat=noScrat, nThreads=nThreads, \
avgTime=parms["mednAvgTime"], avgFreq=parms["mednAvgFreq"], chAvg= parms["mednChAvg"], \
timeWind=parms["mednTimeWind"],flagVer=2, flagTab=2,flagSig=parms["mednSigma"], \
logfile=logFile, check=check, debug=False)
if retCode!=0:
raise RuntimeError("Error in MednFlag")
# Median window frequency editing, for RFI impulsive in frequency
if parms["doFD1"]:
mess = "Median window frequency editing, for RFI impulsive in frequency:"
printMess(mess, logFile)
retCode = EVLAAutoFlag (uv, clist, err, flagVer=2, flagTab=2, doCalib=-1, doBand=-1, \
timeAvg=parms["FD1TimeAvg"], \
doFD=True, FDmaxAmp=1.0e20, FDmaxV=1.0e20, FDwidMW=parms["FD1widMW"], \
FDmaxRMS=[1.0e20,0.1], FDmaxRes=parms["FD1maxRes"], \
FDmaxResBL= parms["FD1maxRes"], FDbaseSel=parms["FD1baseSel"],\
nThreads=nThreads, logfile=logFile, check=check, debug=debug)
if retCode!=0:
raise RuntimeError("Error in AutoFlag")
# Parallactic angle correction?
if parms["doPACor"]:
retCode = EVLAPACor(uv, err, \
logfile=logFile, check=check, debug=debug)
if retCode!=0:
raise RuntimeError("Error in Parallactic angle correction")
# Need to find a reference antenna? See if we have saved it?
if (parms["refAnt"]<=0):
refAnt = FetchObject(fileRoot+".refAnt.pickle")
if refAnt:
parms["refAnt"] = refAnt
# Use bandpass calibrator and center half of each spectrum
if parms["refAnt"]<=0:
mess = "Find best reference antenna: run Calib on BP Cal(s) "
printMess(mess, logFile)
parms["refAnt"] = EVLAGetRefAnt(uv, parms["BPCals"], err, flagVer=0, \
solInt=parms["bpsolint1"], nThreads=nThreads, \
logfile=logFile, check=check, debug=debug)
if err.isErr:
raise RuntimeError("Error finding reference antenna")
if parms["refAnts"][0]<=0:
parms["refAnts"][0] = parms["refAnt"]
mess = "Picked reference antenna "+str(parms["refAnt"])
printMess(mess, logFile)
# Save it
ParmsPicklefile = fileRoot+".Parms.pickle" # Where results saved
SaveObject(parms, ParmsPicklefile, True)
refAntPicklefile = fileRoot+".refAnt.pickle" # Where results saved
SaveObject(parms["refAnt"], refAntPicklefile, True)
# Plot Raw, edited data?
parms["doRawSpecPlot"]=False
parms["doSpecPlot"]=False
if parms["doRawSpecPlot"] and parms["plotSource"]:
mess = "Raw Spectral plot for: "+' '.join(parms["BPCal"])
printMess(mess, logFile)
plotFile = fileRoot+"_RawSpec.ps"
retCode = EVLASpectrum(uv, parms["BPCal"], parms["plotTime"], maxgap, plotFile, parms["refAnt"], err, \
Stokes=["RR","LL"], doband=-1, \
check=check, debug=debug, logfile=logFile )
if retCode!=0:
raise RuntimeError("Error in Plotting spectrum")
EVLAAddOutFile(plotFile, 'project', 'Pipeline log file' )
# delay calibration
if parms["doDelayCal"] and parms["DCals"] and not check:
plotFile = fileRoot+"_DelayCal.ps"
retCode = EVLADelayCal(uv, parms["DCals"], err, \
BChan=parms["delayBChan"], EChan=parms["delayEChan"], \
doCalib=2, flagVer=0, doBand=-1, \
solInt=parms["delaySolInt"], smoTime=parms["delaySmoo"], \
refAnts=[parms["refAnt"]], doTwo=parms["doTwo"],
doZeroPhs=parms["delayZeroPhs"], \
doPlot=parms["doSNPlot"], plotFile=plotFile, \
nThreads=nThreads, noScrat=noScrat, \
logfile=logFile, check=check, debug=debug)
if retCode!=0:
raise RuntimeError("Error in delay calibration")
# Plot corrected data?
if parms["doSpecPlot"] and parms["plotSource"]:
plotFile = fileRoot+"_DelaySpec.ps"
retCode = EVLASpectrum(uv, parms["BPCal"], parms["plotTime"], maxgap, \
plotFile, parms["refAnt"], err, \
Stokes=["RR","LL"], doband=-1, \
check=check, debug=debug, logfile=logFile )
if retCode!=0:
raise RuntimeError("Error in Plotting spectrum")
print(parms["bpBChan1"],parms["bpEChan1"],parms["bpBChan2"],parms["bpEChan2"],parms["bpChWid2"])
# Bandpass calibration
if parms["doBPCal"] and parms["BPCals"]:
retCode = KATBPCal(uv, parms["BPCals"], err, noScrat=noScrat, solInt1=parms["bpsolint1"], \
solInt2=parms["bpsolint2"], solMode=parms["bpsolMode"], \
BChan1=parms["bpBChan1"], EChan1=parms["bpEChan1"], \
BChan2=parms["bpBChan2"], EChan2=parms["bpEChan2"], ChWid2=parms["bpChWid2"], \
doCenter1=parms["bpDoCenter1"], refAnt=parms["refAnt"], \
UVRange=parms["bpUVRange"], doCalib=2, gainUse=0, flagVer=0, doPlot=False, \
nThreads=nThreads, logfile=logFile, check=check, debug=debug)
if retCode!=0:
raise RuntimeError("Error in Bandpass calibration")
# Plot corrected data?
if parms["doSpecPlot"] and parms["plotSource"]:
plotFile = fileRoot+"_BPSpec.ps"
retCode = EVLASpectrum(uv, parms["BPCal"], parms["plotTime"], maxgap, plotFile, \
parms["refAnt"], err, Stokes=["RR","LL"], doband=1, \
check=check, debug=debug, logfile=logFile )
if retCode!=0:
raise RuntimeError("Error in Plotting spectrum")
# Amp & phase Calibrate
if parms["doAmpPhaseCal"]:
plotFile = fileRoot+"_APCal.ps"
retCode = KATCalAP (uv, [], parms["ACals"], err, PCals=parms["PCals"],
doCalib=2, doBand=1, BPVer=1, flagVer=0, \
BChan=parms["ampBChan"], EChan=parms["ampEChan"], \
solInt=parms["solInt"], solSmo=parms["solSmo"], ampScalar=parms["ampScalar"], \
doAmpEdit=parms["doAmpEdit"], ampSigma=parms["ampSigma"], \
ampEditFG=parms["ampEditFG"], \
doPlot=parms["doSNPlot"], plotFile=plotFile, refAnt=parms["refAnt"], \
nThreads=nThreads, noScrat=noScrat, logfile=logFile, check=check, debug=debug)
#print parms["ACals"],parms["PCals"]
if retCode!=0:
raise RuntimeError("Error calibrating")
# More editing
if parms["doAutoFlag"]:
mess = "Post calibration editing:"
printMess(mess, logFile)
# if going to redo then only calibrators
if parms["doRecal"]:
# Only calibrators
clist = []
for DCal in parms["DCals"]:
if DCal["Source"] not in clist:
clist.append(DCal["Source"])
for PCal in parms["PCals"]:
if PCal["Source"] not in clist:
clist.append(PCal["Source"])
for ACal in parms["ACals"]:
if ACal["Source"] not in clist:
clist.append(ACal["Source"])
else:
clist = []
retCode = EVLAAutoFlag (uv, clist, err, flagVer=0, flagTab =2, \
doCalib=2, gainUse=0, doBand=1, BPVer=1, \
IClip=parms["IClip"], minAmp=parms["minAmp"], timeAvg=parms["timeAvg"], \
doFD=parms["doAFFD"], FDmaxAmp=parms["FDmaxAmp"], FDmaxV=parms["FDmaxV"], \
FDwidMW=parms["FDwidMW"], FDmaxRMS=parms["FDmaxRMS"], \
FDmaxRes=parms["FDmaxRes"], FDmaxResBL=parms["FDmaxResBL"], \
FDbaseSel=parms["FDbaseSel"], \
nThreads=nThreads, logfile=logFile, check=check, debug=debug)
if retCode!=0:
raise RuntimeError("Error in AutoFlag")
# Redo the calibration using new flagging?
if parms["doBPCal2"]==None:
parms["doBPCal2"] = parms["doBPCal"]
if parms["doDelayCal2"]==None:
parms["doDelayCal2"] = parms["doDelayCal2"]
if parms["doAmpPhaseCal2"]==None:
parms["doAmpPhaseCal2"] = parms["doAmpPhaseCal"]
if parms["doAutoFlag2"]==None:
parms["doAutoFlagCal2"] = parms["doAutoFlag"]
if parms["doRecal"]:
mess = "Redo calibration:"
printMess(mess, logFile)
EVLAClearCal(uv, err, doGain=True, doFlag=False, doBP=True, check=check, logfile=logFile)
OErr.printErrMsg(err, "Error resetting calibration")
# Parallactic angle correction?
if parms["doPACor"]:
retCode = EVLAPACor(uv, err, \
logfile=logFile, check=check, debug=debug)
if retCode!=0:
raise RuntimeError("Error in Parallactic angle correction")
# Delay recalibration
if parms["doDelayCal2"] and parms["DCals"] and not check:
plotFile = fileRoot+"_DelayCal2.ps"
retCode = EVLADelayCal(uv, parms["DCals"], err, \
BChan=parms["delayBChan"], EChan=parms["delayEChan"], \
doCalib=2, flagVer=0, doBand=-1, \
solInt=parms["delaySolInt"], smoTime=parms["delaySmoo"], \
refAnts=[parms["refAnt"]], doTwo=parms["doTwo"], \
doZeroPhs=parms["delayZeroPhs"], \
doPlot=parms["doSNPlot"], plotFile=plotFile, \
nThreads=nThreads, noScrat=noScrat, \
logfile=logFile, check=check, debug=debug)
if retCode!=0:
raise RuntimeError("Error in delay calibration")
# Plot corrected data?
if parms["doSpecPlot"] and parms["plotSource"]:
plotFile = fileRoot+"_DelaySpec2.ps"
retCode = EVLASpectrum(uv, parms["BPCal"], parms["plotTime"], maxgap, plotFile, parms["refAnt"], err, \
Stokes=["RR","LL"], doband=-1, \
check=check, debug=debug, logfile=logFile )
if retCode!=0:
raise RuntimeError("Error in Plotting spectrum")
# Bandpass calibration
if parms["doBPCal2"] and parms["BPCals"]:
retCode = KATBPCal(uv, parms["BPCals"], err, noScrat=noScrat, solInt1=parms["bpsolint1"], \
solInt2=parms["bpsolint2"], solMode=parms["bpsolMode"], \
BChan1=parms["bpBChan1"], EChan1=parms["bpEChan1"], \
BChan2=parms["bpBChan2"], EChan2=parms["bpEChan2"], ChWid2=parms["bpChWid2"], \
doCenter1=parms["bpDoCenter1"], refAnt=parms["refAnt"], \
UVRange=parms["bpUVRange"], doCalib=2, gainUse=0, flagVer=0, doPlot=False, \
nThreads=nThreads, logfile=logFile, check=check, debug=debug)
if retCode!=0:
raise RuntimeError("Error in Bandpass calibration")
# Plot corrected data?
if parms["doSpecPlot"] and parms["plotSource"]:
plotFile = fileRoot+"_BPSpec2.ps"
retCode = EVLASpectrum(uv, parms["BPCal"], parms["plotTime"], maxgap, plotFile, parms["refAnt"], err, \
Stokes=["RR","LL"], doband=1, \
check=check, debug=debug, logfile=logFile )
if retCode!=0:
raise RuntimeError("Error in Plotting spectrum")
# Amp & phase Recalibrate
if parms["doAmpPhaseCal2"]:
plotFile = fileRoot+"_APCal2.ps"
retCode = KATCalAP (uv, [], parms["ACals"], err, PCals=parms["PCals"], \
doCalib=2, doBand=2, BPVer=1, flagVer=0, \
BChan=parms["ampBChan"], EChan=parms["ampEChan"], \
solInt=parms["solInt"], solSmo=parms["solSmo"], ampScalar=parms["ampScalar"], \
doAmpEdit=True, ampSigma=parms["ampSigma"], \
ampEditFG=parms["ampEditFG"], \
doPlot=parms["doSNPlot"], plotFile=plotFile, refAnt=parms["refAnt"], \
noScrat=noScrat, nThreads=nThreads, logfile=logFile, check=check, debug=debug)
if retCode!=0:
raise RuntimeError("Error calibrating")
# More editing
if parms["doAutoFlag2"]:
mess = "Post recalibration editing:"
printMess(mess, logFile)
retCode = EVLAAutoFlag (uv, [], err, flagVer=0, flagTab=2, \
doCalib=2, gainUse=0, doBand=1, BPVer=1, \
IClip=parms["IClip"], minAmp=parms["minAmp"], timeAvg=parms["timeAvg"], \
doFD=parms["doAFFD"], FDmaxAmp=parms["FDmaxAmp"], FDmaxV=parms["FDmaxV"], \
FDwidMW=parms["FDwidMW"], FDmaxRMS=parms["FDmaxRMS"], \
FDmaxRes=parms["FDmaxRes"], FDmaxResBL= parms["FDmaxResBL"], \
FDbaseSel=parms["FDbaseSel"], \
nThreads=nThreads, logfile=logFile, check=check, debug=debug)
if retCode!=0:
raise RuntimeError("Error in AutoFlag")
# end recal
# Calibrate and average data
if parms["doCalAvg"]:
retCode = KATCalAvg (uv, avgClass, parms["seq"], parms["CalAvgTime"], err, \
flagVer=2, doCalib=2, gainUse=0, doBand=1, BPVer=1, doPol=False, \
avgFreq=parms["avgFreq"], chAvg=parms["chAvg"], \
BChan=1, EChan=parms["selChan"] - 1, doAuto=parms["doAuto"], \
BIF=parms["CABIF"], EIF=parms["CAEIF"], Compress=parms["Compress"], \
nThreads=nThreads, logfile=logFile, check=check, debug=debug)
if retCode!=0:
raise RuntimeError("Error in CalAvg")
# Get calibrated/averaged data
if not check:
uv = UV.newPAUV("AIPS UV DATA", EVLAAIPSName(project), avgClass[0:6], \
disk, parms["seq"], True, err)
if err.isErr:
OErr.printErrMsg(err, "Error creating cal/avg AIPS data")
KATUVFITS(uv, 'preimage.uvfits', 0, err, exclude=["AIPS HI", "AIPS SL", "AIPS PL"], include=["AIPS AN", "AIPS FQ"], compress=parms["Compress"], logfile=logFile)
KATUVFITab(uv, 'preimage.uvtab', 0, err)
# XClip
if parms["XClip"] and parms["XClip"]>0.0:
mess = "Cross Pol clipping:"
printMess(mess, logFile)
retCode = EVLAAutoFlag (uv, [], err, flagVer=-1, flagTab=1, \
doCalib=2, gainUse=0, doBand=-1, maxBad=1.0, \
XClip=parms["XClip"], timeAvg=1./60., \
nThreads=nThreads, logfile=logFile, check=check, debug=debug)
if retCode!=0:
raise RuntimeError("Error in AutoFlag")
# R-L delay calibration cal if needed,
if parms["doRLDelay"] and parms["RLDCal"][0][0]!=None:
if parms["rlrefAnt"]<=0:
parms["rlrefAnt"] = parms["refAnt"]
# parms["rlDoBand"] if before average, BPVer=parms["rlBPVer"],
retCode = EVLARLDelay(uv, err,\
RLDCal=parms["RLDCal"], BChan=parms["rlBChan"], \
EChan=parms["rlEChan"], UVRange=parms["rlUVRange"], \
soucode=parms["rlCalCode"], doCalib=parms["rlDoCal"], gainUse=parms["rlgainUse"], \
timerange=parms["rltimerange"], \
# NOT HERE doBand=parms["rlDoBand"], BPVer=parms["rlBPVer"], \
flagVer=parms["rlflagVer"], \
refAnt=parms["rlrefAnt"], doPol=False, \
nThreads=nThreads, noScrat=noScrat, logfile=logFile, \
check=check, debug=debug)
if retCode!=0:
raise RuntimeError("Error in R-L delay calibration")
# Polarization calibration
if parms["doPolCal"]:
if parms["PCRefAnt"]<=0:
parms["PCRefAnt"] = parms["refAnt"]
retCode = EVLAPolCal(uv, parms["PCInsCals"], err, \
doCalib=2, gainUse=0, doBand=-1, flagVer=0, \
fixPoln=parms["PCFixPoln"], pmodel=parms["PCpmodel"], avgIF=parms["PCAvgIF"], \
solInt=parms["PCSolInt"], refAnt=parms["PCRefAnt"], solType=parms["PCSolType"], \
ChInc=parms["PCChInc"], ChWid=parms["PCChWid"], \
nThreads=nThreads, check=check, debug=debug, noScrat=noScrat, logfile=logFile)
if retCode!=0 and (not check):
raise RuntimeError("Error in polarization calibration: "+str(retCode))
# end poln cal.
# R-L phase calibration cal., creates new BP table
if parms["doRLCal"] and parms["RLDCal"][0][0]!=None:
plotFile = fileRoot+"_RLSpec2.ps"
if parms["rlrefAnt"]<=0:
parms["rlrefAnt"] = parms["refAnt"]
retCode = EVLARLCal(uv, err,\
RLDCal=parms["RLDCal"], BChan=parms["rlBChan"],
EChan=parms["rlEChan"], UVRange=parms["rlUVRange"], \
ChWid2=parms["rlChWid"], solInt1=parms["rlsolint1"], solInt2=parms["rlsolint2"], \
RLPCal=parms["RLPCal"], RLPhase=parms["RLPhase"], \
RM=parms["RLRM"], CleanRad=parms["rlCleanRad"], \
calcode=parms["rlCalCode"], doCalib=parms["rlDoCal"], gainUse=parms["rlgainUse"], \
timerange=parms["rltimerange"], FOV=parms["rlFOV"], \
doBand=-1, BPVer=1, flagVer=parms["rlflagVer"], \
refAnt=parms["rlrefAnt"], doPol=parms["doPol"], PDVer=parms["PDVer"], \
doPlot=parms["doSpecPlot"], plotFile=plotFile, \
nThreads=nThreads, noScrat=noScrat, logfile=logFile, \
check=check, debug=debug)
if retCode!=0:
raise RuntimeError("Error in RL phase spectrum calibration")
# VClip
if parms["VClip"] and parms["VClip"]>0.0:
mess = "VPol clipping:"
printMess(mess, logFile)
retCode = EVLAAutoFlag (uv, [], err, flagVer=-1, flagTab=1, \
doCalib=2, gainUse=0, doBand=-1, \
VClip=parms["VClip"], timeAvg=parms["timeAvg"], \
nThreads=nThreads, logfile=logFile, check=check, debug=debug)
if retCode!=0:
raise RuntimeError("Error in AutoFlag VClip")
# Plot corrected data?
parms["doSpecPlot"]=True
if parms["doSpecPlot"] and parms["plotSource"]:
plotFile = fileRoot+"_Spec.ps"
retCode = EVLASpectrum(uv, parms["BPCal"], parms["plotTime"], maxgap, \
plotFile, parms["refAnt"], err, \
Stokes=["I"], doband=-1, \
check=check, debug=debug, logfile=logFile )
if retCode!=0:
raise RuntimeError("Error in Plotting spectrum")
# Image targets
if parms["doImage"]:
# If targets not specified, image all
if len(parms["targets"])<=0:
slist = EVLAAllSource(uv,err,logfile=logFile,check=check,debug=debug)
else:
slist = targets
slist=targets
KATImageTargets (uv, err, Sources=slist, seq=parms["seq"], sclass=outIClass, OutlierArea=parms["outlierArea"],\
doCalib=-1, doBand=-1, flagVer=-1, doPol=parms["doPol"], PDVer=parms["PDVer"], \
Stokes=parms["Stokes"], FOV=parms["FOV"], Robust=parms["Robust"], Niter=parms["Niter"], \
CleanRad=parms["CleanRad"], minFlux=parms["minFlux"], OutlierSize=parms["OutlierSize"], \
xCells=parms["xCells"], yCells=parms["yCells"], Reuse=parms["Reuse"], minPatch=parms["minPatch"], \
maxPSCLoop=parms["maxPSCLoop"], minFluxPSC=parms["minFluxPSC"], noNeg=parms["noNeg"], \
solPInt=parms["solPInt"], solPMode=parms["solPMode"], solPType=parms["solPType"], \
maxASCLoop=parms["maxASCLoop"], minFluxASC=parms["minFluxASC"], nx=parms["nx"], ny=parms["ny"], \
solAInt=parms["solAInt"], solAMode=parms["solAMode"], solAType=parms["solAType"], \
avgPol=parms["avgPol"], avgIF=parms["avgIF"], minSNR = parms["minSNR"], refAnt=parms["refAnt"], \
do3D=parms["do3D"], BLFact=parms["BLFact"], BLchAvg=parms["BLchAvg"], \
doMB=parms["doMB"], norder=parms["MBnorder"], maxFBW=parms["MBmaxFBW"], \
PBCor=parms["PBCor"],antSize=parms["antSize"], autoCen=parms["autoCen"], \
nTaper=parms["nTaper"], Tapers=parms["Tapers"], sefd=sefd, \
nThreads=nThreads, noScrat=noScrat, logfile=logFile, check=check, debug=False)
# End image
# Get report on sources
if parms["doReport"]:
# If targets not specified, do all
if len(parms["targets"])<=0:
slist = EVLAAllSource(uv,err,logfile=logFile,check=check,debug=debug)
else:
slist = parms["targets"]
Report = EVLAReportTargets(uv, err, Sources=slist, seq=parms["seq"], sclass=outIClass, \
Stokes=parms["Stokes"], logfile=logFile, check=check, debug=debug)
# Save to pickle jar
ReportPicklefile = fileRoot+"_Report.pickle" # Where results saved
SaveObject(Report, ReportPicklefile, True)
# Write results, cleanup
# Save cal/average UV data?
if parms["doSaveUV"] and (not check):
Aname = EVLAAIPSName(project)
cno = AIPSDir.PTestCNO(disk, user, Aname, avgClass[0:6], "UV", parms["seq"], err)
if cno>0:
uvt = UV.newPAUV("AIPS CAL UV DATA", Aname, avgClass, disk, parms["seq"], True, err)
filename = fileRoot+"_Cal.uvtab"
KATUVFITS (uv, filename, 0, err, exclude=["AIPS HI", "AIPS SL", "AIPS PL"], include=["AIPS AN", "AIPS FQ"], compress=parms["Compress"], logfile=logFile)
EVLAAddOutFile(os.path.basename(filename), 'project', "Calibrated Averaged UV data" )
# Save list of output files
EVLASaveOutFiles(manifestfile)
del uvt
# Imaging results
# If targets not specified, save all
if len(parms["targets"])<=0:
slist = EVLAAllSource(uv,err,logfile=logFile,check=check,debug=debug)
else:
slist = parms["targets"]
for target in slist:
if parms["doSaveImg"] and (not check):
for s in parms["Stokes"]:
oclass = s+outIClass[1:]
outname = target
# Test if image exists
cno = AIPSDir.PTestCNO(disk, user, outname, oclass, "MA", parms["seq"], err)
#print cno
if cno <= 0 :
continue
x = Image.newPAImage("out", outname, oclass, disk, parms["seq"], True, err)
outfilefits = fileRoot+'_'+target+"."+oclass+".fits"
xf = KATImFITS(x, outfilefits, 0, err, logfile=logFile)
x = Image.newPAImage("out", outname, oclass, disk, parms["seq"], True, err)
outfile = fileRoot+'_'+target+"."+oclass+".fittab.fits"
xf = EVLAImFITS (x, outfile, 0, err, logfile=logFile)
EVLAAddOutFile(outfile, target, 'Image of '+ target)
# Statistics
zz=imstat(x, err, logfile=logFile)
# Make a Jpeg image
FITS2jpeg.fits2jpeg(outfilefits,chans=1,contrast=0.05,cmap='jet',area=0.7)
EVLAAddOutFile(outfile.replace('.fits','.jpeg'), target, 'Jpeg image of '+ target)
# end writing loop
# Save list of output files
EVLASaveOutFiles(manifestfile)
OErr.printErrMsg(err, "Writing output")
# Contour plots
if parms["doKntrPlots"]:
mess = "INFO --> Contour plots (doKntrPlots)"
printMess(mess, logFile)
EVLAKntrPlots( err, imName=parms["targets"], project=fileRoot,
disk=disk, debug=debug )
# Save list of output files
EVLASaveOutFiles(manifestfile)
elif debug:
mess = "Not creating contour plots ( doKntrPlots = "+str(parms["doKntrPlots"])+ " )"
printMess(mess, logFile)
# Source uv plane diagnostic plots
if parms["doDiagPlots"]:
mess = "INFO --> Diagnostic plots (doDiagPlots)"
printMess(mess, logFile)
# Get the highest number avgClass catalog file
Aname = EVLAAIPSName( project )
uvc = None
if not check:
uvname = project+"_Cal"
uvc = UV.newPAUV(uvname, Aname, avgClass, disk, parms["seq"], True, err)
EVLADiagPlots( uvc, err, cleanUp=parms["doCleanup"], \
project=fileRoot, \
logfile=logFile, check=check, debug=debug )
# Save list of output files
EVLASaveOutFiles(manifestfile)
elif debug:
mess = "Not creating diagnostic plots ( doDiagPlots = "+str(parms["doDiagPlots"])+ " )"
printMess(mess, logFile)
# Save metadata
srcMetadata = None
projMetadata = None
if parms["doMetadata"]:
mess = "INFO --> Save metadata (doMetadata)"
printMess(mess, logFile)
uvc = None
if not uvc:
# Get calibrated/averaged data
Aname = EVLAAIPSName(project)
uvname = project+"_Cal"
uvc = UV.newPAUV(uvname, Aname, avgClass, disk, parms["seq"], True, err)
if err.isErr:
OErr.printErrMsg(err, "Error creating cal/avg AIPS data")
# Get source metadata; save to pickle file
srcMetadata = EVLASrcMetadata( uvc, err, Sources=parms["targets"], seq=parms["seq"], \
sclass=outIClass, Stokes=parms["Stokes"],\
logfile=logFile, check=check, debug=debug )
picklefile = fileRoot+".SrcReport.pickle"
SaveObject( srcMetadata, picklefile, True )
EVLAAddOutFile(os.path.basename(picklefile), 'project', 'All source metadata' )
# Get project metadata; save to pickle file
projMetadata = KATProjMetadata( uvc, AIPS_VERSION, err, \
PCals=parms["PCals"], ACals=parms["ACals"], \
BPCals=parms["BPCals"], DCals=parms["DCals"], \
project = project, band = band, \
dataInUVF = parms["archRoot"], archFileID = fileRoot )
picklefile = fileRoot+".ProjReport.pickle"
SaveObject(projMetadata, picklefile, True)
EVLAAddOutFile(os.path.basename(picklefile), 'project', 'Project metadata' )
else:
# Fetch from pickle jar
picklefile = fileRoot+".SrcReport.pickle"
srcMetadata = FetchObject(picklefile)
picklefile = fileRoot+".ProjReport.pickle"
projMetadata = FetchObject(picklefile)
# Write report
if parms["doHTML"]:
mess = "INFO --> Write HTML report (doHTML)"
printMess(mess, logFile)
KATHTMLReport( projMetadata, srcMetadata, \
outfile=fileRoot+"_report.html", \
logFile=logFile )
# Write VOTable
if parms["doVOTable"]:
mess = "INFO --> Write VOTable (doVOTable)"
printMess(mess, logFile)
EVLAAddOutFile( 'VOTable.xml', 'project', 'VOTable report' )
EVLAWriteVOTable( projMetadata, srcMetadata, filename=fileRoot+'_VOTable.xml' )
# Save list of output files
EVLASaveOutFiles(manifestfile)
# Cleanup - delete AIPS files
if parms["doCleanup"] and (not check):
mess = "INFO --> Clean up (doCleanup)"
printMess(mess, logFile)
# Delete target images
# How many Stokes images
nstok = len(parms["Stokes"])
for istok in range(0,nstok):
oclass = parms["Stokes"][istok:istok+1]+outIClass[1:]
AllDest(err, disk=disk,Aseq=parms["seq"],Aclass=oclass)
# Delete initial UV data
Aname = EVLAAIPSName(project)
# Test if data exists
cno = AIPSDir.PTestCNO(disk, user, Aname, dataClass[0:6], "UV", parms["seq"], err)
if cno>0:
uvt = UV.newPAUV("AIPS RAW UV DATA", Aname, dataClass[0:6], disk, parms["seq"], True, err)
uvt.Zap(err)
del uvt
if err.isErr:
OErr.printErrMsg(err, "Error deleting raw AIPS data")
# Zap calibrated/averaged data
# Test if data exists
cno = AIPSDir.PTestCNO(disk, user, Aname, avgClass[0:6], "UV", parms["seq"], err)
if cno>0:
uvt = UV.newPAUV("AIPS CAL UV DATA", Aname, avgClass[0:6], disk, parms["seq"], True, err)
uvt.Zap(err)
del uvt
if err.isErr:
OErr.printErrMsg(err, "Error deleting cal/avg AIPS data")
# Zap UnHanned data if present
loadClass = "Raw"
# Test if image exists
cno = AIPSDir.PTestCNO(disk, user, Aname, loadClass[0:6], "UV", parms["seq"], err)
if cno>0:
uvt = UV.newPAUV("AIPS CAL UV DATA", Aname, loadClass[0:6], disk, parms["seq"], True, err)
uvt.Zap(err)
del uvt
if err.isErr:
OErr.printErrMsg(err, "Error deleting cal/avg AIPS data")
OErr.printErrMsg(err, "Writing output/cleanup")
# Delete AIPS scratch DA00 and disk
if os.path.exists(os.environ['DA00']): shutil.rmtree(os.environ['DA00'])
for disk in ObitTalkUtil.AIPSDir.AIPSdisks:
if os.path.exists(disk): shutil.rmtree(disk)
# Shutdown
mess = "Finished project "+parms["project"]+ \
" AIPS user no. "+str(AIPS.userno)
printMess(mess, logFile)
OErr.printErr(err)
OSystem.Shutdown(ObitSys)
def pipeline(aipsSetup, parmFile):
"""
VLBA Continuum pipeline.
* *aipsSetup* = AIPS setup file
* *parmFile* = pipeline input parameters file
"""
############################# Initialize OBIT ##########################################
noScrat = []
exec(open(aipsSetup).read())
VLBAAddOutFile(aipsSetup, 'project', "Obit's AIPS setup file")
############################# Default parameters ##########################################
# Define data
project = "Unspecified" # Project name (12 char or less, used as AIPS Name)
session = "?" # Project session code
band = "?" # Observing band
logFile = project + "_" + session + "_" + band + ".log" # Processing log file
seq = 1 # AIPS sequence number
gain = 0.10 # CLEAN loop gain
doLoadIDI = True # Load data from IDI FITS?
doLoadUVF = False # Load the "AIPS Friendly" (uvfits) FITS version
dataInUVF = None # Input uvfits data file name
dataInIDI = None # Input FITS-IDI file or list
dataClass = "Raw" # AIPS class of raw uv data
Compress = False # Use compressed UV data?
calInt = 0.15 # Calibration table interval in min.
wtThresh = 0.8 # Data weight threshold
check = False # Only check script, don't execute tasks
debug = False # run tasks debug
prtLv = 2 # Print level
# Initialize parameters
parms = VLBAInitContParms()
############################# Set Project Processing parameters ##################
exec(open(parmFile).read())
VLBAAddOutFile(parmFile, 'project', 'Pipeline input parameters')
################################## Process #####################################
# Init cal pickle jars
goodCalPicklefile = project + "_" + session + "_" + band + "_GoodCal.pickle" # Where results saved
# Default "best" calibration
goodCal = {
"Source": " ",
"souID": 0,
"timeRange": (0.0, 100.0),
"Fract": 0.0,
"SNR": 0.0,
"bestRef": 0
}
# Save initial default if it doesn't already exist
SaveObject(goodCal, goodCalPicklefile, False)
VLBAAddOutFile(goodCalPicklefile, 'project', 'Best calibrator information')
OKCalPicklefile = project + "_" + session + "_" + band + "_OKCal.pickle" # Where results saved
SaveObject(parms["contCals"], OKCalPicklefile, False)
VLBAAddOutFile(OKCalPicklefile, 'project', 'List of calibrators')
targetsPicklefile = project + "_" + session + "_" + band + "_targets.pickle" # Where results saved
if (not parms["targets"]) and os.path.exists(targetsPicklefile):
parms["targets"] = FetchObject(targetsPicklefile)
else:
SaveObject(parms["targets"], targetsPicklefile, False)
VLBAAddOutFile(targetsPicklefile, 'project', 'List of targets')
# Load the outputs pickle jar
VLBAFetchOutFiles()
# Logging directly to logFile
OErr.PInit(err, prtLv, logFile)
retCode = 0
VLBAAddOutFile(logFile, 'project', 'Pipeline log file')
mess = "Start project " + project + " session " + session + " " + band + " Band" + " AIPS user no. " + str(
AIPS.userno)
printMess(mess, logFile)
logger.info(mess)
if debug:
pydoc.ttypager = pydoc.plainpager # don't page task input displays
mess = "Using Debug mode "
printMess(mess, logFile)
if check:
mess = "Only checking script"
printMess(mess, logFile)
# Load Data from FITS
uv = None # Raw data
uvc = None # Cal/averaged data
if doLoadIDI:
logger.info("--> Load IDI data file (doLoadIDI)")
if type(dataInIDI) == list:
logger.info(
"--> Loading a list of IDI files (dataInIDI is a list)\n" +
" This data appears to be from the old correlator.\n" +
" IDI corrections will be applied when loading is\n" +
" complete.")
# Read list
for dataIn in dataInIDI:
uv = VLBAIDILoad(dataIn, project, session, band, dataClass, disk, seq, err, logfile=logFile, \
wtThresh=wtThresh, calInt=calInt, Compress=Compress, \
check=check, debug=debug)
if not UV.PIsA(uv):
raise RuntimeError("Cannot load " + dataIn)
# Fix IDI files:
uv = IDIFix.IDIFix(uv, err)
seq = uv.Aseq
else:
logger.info(
"--> Loading a single IDI file (dataInIDI is not a list)\n" +
" This data appears to be from the DiFX correlator.\n" +
" No IDI corrections will be applied.")
# Single IDI file
uv = VLBAIDILoad(dataInIDI, project, session, band, dataClass, disk, seq, err, logfile=logFile, \
wtThresh=wtThresh, calInt=calInt, Compress=Compress, \
check=check, debug=debug)
if not UV.PIsA(uv):
raise RuntimeError("Cannot load " + dataInIDI)
if doLoadUVF:
logger.info("--> Load UVFITS data file (doLoadUVF)")
uv = VLBAIDILoad(dataInUVF, project, session, band, dataClass, disk, seq, err, logfile=logFile, \
wtThresh=wtThresh, calInt=calInt, Compress=Compress, \
check=check, debug=debug)
# Adding check condition to avoid error when checking
if not UV.PIsA(uv) and not check:
raise RuntimeError("Cannot load " + dataInUVF)
# Otherwise set uv
if uv == None and not check:
Aname = VLBAAIPSName(project, session)
uvname = project + "_" + session + "_" + band
uv = UV.newPAUV(uvname, Aname, dataClass, disk, seq, True, err)
if err.isErr:
OErr.printErrMsg(err, "Error creating AIPS data")
# frequency dependent default parameters
VLBAInitContFQParms(uv, parms, err, \
logfile=logFile, check=check, debug=debug)
# Setup target source list
if os.path.exists(targetsPicklefile):
parms["targets"] = \
setupSourceList( parms["targets"], uv, err, logFile, check, debug )
logger.debug("parms['targets'] = " + str(parms["targets"]))
# Clear any old calibration/editing
if parms["doClearTab"]:
logger.info("--> Clear old calibration/editing (doClearTab)")
VLBAClearCal(uv, err, doGain=parms["doGain"], doFlag=parms["doFlag"], doBP=parms["doBP"], \
check=check, logfile=logFile)
OErr.printErrMsg(err, "Error resetting calibration")
# Copy FG 1 to FG 2
if parms["doCopyFG"]:
logger.info("--> Copy flag (FG) table 1 to flag table 2 (doCopyFG)")
retCode = VLBACopyFG(uv,
err,
logfile=logFile,
check=check,
debug=debug)
if retCode != 0:
raise RuntimeError("Error Copying FG table")
# Special editing
if parms["doEditList"] and not check:
logger.info("--> Special editing (doEditList)")
for edt in parms["editList"]:
UV.PFlag(uv,err,timeRange=[dhms2day(edt["timer"][0]),dhms2day(edt["timer"][1])], \
flagVer=editFG, Ants=edt["Ant"], Chans=edt["Chans"], IFs=edt["IFs"], \
Stokes=edt["Stokes"], Reason=edt["Reason"])
OErr.printErrMsg(err, "Error Flagging")
# Quack to remove data from start and end of each scan
if parms["doQuack"]:
logger.info("--> Remove data from start and end of scans (doQuack)")
retCode = VLBAQuack (uv, err, \
begDrop=parms["quackBegDrop"], endDrop=parms["quackEndDrop"], \
Reason=parms["quackReason"], \
logfile=logFile, check=check, debug=debug)
if retCode != 0:
raise RuntimeError("Error Quacking data")
# Median window time editing, for RFI impulsive in time or dropouts
if parms["doMedn"]:
logger.info("--> Median window time editing (doMedn)")
retCode = VLBAMedianFlag (uv, " ", err, noScrat=noScrat, nThreads=nThreads, \
avgTime=parms["mednAvgTime"], avgFreq=parms["mednAvgFreq"], \
chAvg=parms["mednChAvg"], timeWind=parms["mednTimeWind"], \
flagSig=parms["mednSigma"], flagVer=2, \
logfile=logFile, check=check, debug=False)
if retCode != 0:
raise RuntimeError("Error in MednFlag")
# Quantization correction?
if parms["doQuantCor"]:
logger.info("--> Quantization correction (doQuantCor)")
plotFile = project + "_" + session + "_" + band + ".Quant.ps"
retCode = VLBAQuantCor(uv, parms["QuantSmo"], parms["QuantFlag"], err, \
doSNPlot=parms["doSNPlot"], plotFile=plotFile, \
logfile=logFile, check=check, debug=debug)
if retCode != 0:
raise RuntimeError("Error in quantization correcting/flagging")
# Parallactic angle correction?
if parms["doPACor"]:
logger.info("--> Parallactic angle correction (doPACor)")
retCode = VLBAPACor(uv, err, noScrat=noScrat, \
logfile=logFile, check=check, debug=debug)
if retCode != 0:
raise RuntimeError("Error in quantization correcting/flagging")
# Opacity/Tsys/gain correction
if parms["doOpacCor"]:
logger.info("--> Opacity/Tsys/Gain correction (doOpacCor)")
plotFile = project + "_" + session + "_" + band + ".Opacity.ps"
retCode = VLBAOpacCor(uv, parms["OpacSmoo"], err, \
doSNPlot=parms["doSNPlot"], plotFile=plotFile, \
logfile=logFile, check=check, debug=debug)
if retCode != 0:
raise RuntimeError("Error in opacity/gain.Tsys correction")
VLBASaveOutFiles() # Save plot file in Outfiles
# Need to determine a list of calibrators?
if (parms["contCals"] == None) or (len(parms["contCals"]) <= 0):
logger.info("--> Get list of calibrators (contCals = None or [])")
if parms["doFindOK"]:
slist = VLBAAllSource(uv,
err,
logfile=logFile,
check=check,
debug=debug)
parms["contCals"] = VLBAOKCal(uv, parms["minOKFract"], err, \
solInt=parms["findSolInt"], \
calSou=slist, minSNR=parms["minOKSNR"], \
doCalib=-1, flagVer=2, refAnts=parms["refAnts"], \
noScrat=noScrat, nThreads=nThreads, \
logfile=logFile, check=check, debug=debug)
if not parms["contCals"] and not check:
raise RuntimeError("Error in finding acceptable calibrators")
logger.info("Calibrators = " + str(parms["contCals"]))
else:
# Snatch from pickle jar
parms["contCals"] = FetchObject(OKCalPicklefile)
# Save contCals to a pickle jar
SaveObject(parms["contCals"], OKCalPicklefile, True)
# Find best calibration source
if parms["doFindCal"]:
logger.info("--> Find best calibration source (doFindCal)")
goodCal = VLBAGoodCal(uv, err, \
solInt=parms["findSolInt"], timeInt=parms["findTimeInt"], \
calSou=parms["contCals"], \
#CalModel=parms["contCalModel"], \
doCalib=-1, flagVer=2, refAnts=parms["refAnts"], \
noScrat=noScrat, nThreads=nThreads, \
logfile=logFile, check=check, debug=debug)
if not goodCal and not check:
raise RuntimeError("Error in finding best calibration data")
# Save it to a pickle jar
SaveObject(goodCal, goodCalPicklefile, True)
else:
# Fetch from pickle
goodCal = FetchObject(goodCalPicklefile)
# Apply Phase cals from PC table?
if parms["doPCcor"] and not check:
logger.info("--> Apply phase cals (doPCcor)")
plotFile = project + "_" + session + "_" + band + ".PC.ps"
retCode = VLBAPCcor(uv, err, calSou=goodCal["Source"], \
timeRange=goodCal["timeRange"], \
doCalib=-1, flagVer=2, solInt=parms["manPCsolInt"], \
PCin=1, SNout=0, refAnt=goodCal["bestRef"], \
doPCPlot=parms["doPCPlot"], plotFile=plotFile, \
noScrat=noScrat, logfile=logFile, check=check, debug=debug)
if retCode != 0:
raise RuntimeError("Error in PC calibration")
VLBASaveOutFiles() # Save plot file in Outfiles
# manual phase cal
if parms["doManPCal"] and not check:
logger.info("--> Manual phase cal (doManPCal)")
plotFile = project + session + band + ".ManPCal.ps"
retCode = VLBAManPCal(uv, err, calSou=goodCal["Source"], \
#CalModel=parms["contCalModel"], \
timeRange=goodCal["timeRange"], \
solInt=parms["manPCsolInt"], smoTime=parms["manPCSmoo"], \
refAnts=[goodCal["bestRef"]], doCalib=2, flagVer=2, \
doManPCalPlot=parms["doManPCalPlot"], \
plotFile=plotFile, noScrat=noScrat, \
nThreads=nThreads, logfile=logFile, check=check, debug=debug)
if retCode != 0:
raise RuntimeError("Error in manual phase calibration")
# image cals
if parms["doImgCal"] and not check:
logger.info("--> Image calibrators (doImgCal)")
VLBAImageCals(uv,
err,
Sources=parms["contCals"],
seq=seq,
sclass=parms["outCclass"],
doCalib=2,
flagVer=2,
doBand=0,
FOV=parms["FOV"],
Robust=parms["Robust"],
maxPSCLoop=parms["maxPSCLoop"],
minFluxPSC=parms["minFluxPSC"],
solPInt=parms["solPInt"],
solMode=parms["solMode"],
maxASCLoop=parms["maxASCLoop"],
minFluxASC=parms["minFluxASC"],
solAInt=parms["solAInt"],
avgPol=parms["avgPol"],
avgIF=parms["avgIF"],
minSNR=parms["minSNR"],
refAnt=goodCal["bestRef"],
nThreads=nThreads,
noScrat=noScrat,
logfile=logFile,
check=check,
debug=debug)
# Rewrite OKCal pickle file because calibrators may have been updated
SaveObject(parms["contCals"], OKCalPicklefile, True)
if len(parms["contCals"]) <= 0:
logger.error(
"No calibration sources have been detected! Stopping pipeline."
)
raise RuntimeError("No calibration sources have been detected!")
# Check if calibrator models now available
parms["contCalModel"] = VLBAImageModel(parms["contCals"],
parms["outCclass"], disk, seq, err)
# delay calibration
if parms["doDelayCal"] and not check:
logger.info("--> Delay calibration (doDelayCal)")
plotFile = project + "_" + session + "_" + band + ".DelayCal.ps"
retCode = VLBADelayCal(uv, err, calSou=parms["contCals"], CalModel=parms["contCalModel"], \
doCalib=2, flagVer=2, doBand=0, \
solInt=parms["manPCsolInt"], smoTime=parms["delaySmoo"], \
refAnts=[goodCal["bestRef"]], \
doSNPlot=parms["doSNPlot"], plotFile=plotFile, \
nThreads=nThreads, noScrat=noScrat, logfile=logFile, check=check, debug=debug)
if retCode != 0:
raise RuntimeError("Error in delay calibration")
VLBASaveOutFiles() # Save plot file in Outfiles
# Bandpass calibration if needed
if parms["doBPCal"] and not check:
logger.info("--> Bandpass calibration (doBPCal)")
retCode = VLBABPass(uv, goodCal["Source"], err, CalModel=parms["contCalModel"], \
timeRange=goodCal["timeRange"], doCalib=2, flagVer=2, \
noScrat=noScrat, solInt1=parms["bpsolint1"], \
solInt2=parms["bpsolint2"], solMode=parms["bpsolMode"], \
BChan1=parms["bpBChan1"], EChan1=parms["bpEChan1"], BChan2=parms["bpBChan2"], \
EChan2=parms["bpEChan2"], ChWid2=parms["bpChWid2"], \
doCenter1=parms["bpDoCenter1"], refAnt=goodCal["bestRef"], specIndex=parms["specIndex"], \
doAuto = parms["bpdoAuto"], \
nThreads=nThreads, logfile=logFile, check=check, debug=debug)
if retCode != 0:
raise RuntimeError("Error in Bandpass calibration")
# Plot amplitude and phase vs. frequency
if parms["doSpecPlot"]:
logger.info("--> Spectral plotting (doSpecPlot)")
plotFile = project + '_' + session + '_' + band + ".spec.ps"
VLBASpecPlot(uv,
goodCal,
err,
doband=1,
check=check,
plotFile=plotFile,
logfile=logFile,
debug=debug)
VLBASaveOutFiles() # Save plot file in Outfiles
# Phase calibration using calibrator models
if parms["doPhaseCal"]:
logger.info(
"--> Phase calibration using calibrator models (doPhaseCal)")
plotFile = project + "_" + session + "_" + band + ".PhaseCal0.ps"
retCode = VLBAPhaseCal(uv, err, calSou=parms["contCals"], CalModel=parms["contCalModel"], \
doCalib=-1, flagVer=0, doBand=-1, \
refAnt=goodCal["bestRef"], solInt=parms["solPInt"], \
doSNPlot=parms["doSNPlot"], plotFile=plotFile, \
nThreads=nThreads, noScrat=noScrat, logfile=logFile, check=check, debug=debug)
if retCode != 0:
raise RuntimeError("Error in phase calibration")
VLBASaveOutFiles() # Save plot file in Outfiles
# Rewrite contCals pickle file because contCals may have been updated
SaveObject(parms["contCals"], OKCalPicklefile, True)
if len(parms["contCals"]) <= 0:
logger.error(
"No calibrator sources have been detected! Stopping pipeline.")
raise RuntimeError("No calibrator sources have been detected!")
# Amplitude calibration
if parms["doAmpCal"] and not check:
logger.info("--> Amplitude calibration (doAmpCal)")
plotFile = project + "_" + session + "_" + band + ".AmpCal.ps"
retCode = VLBAAmpCal(uv,
err,
calSou=parms["contCals"],
CalModel=parms["contCalModel"],
doCalib=2,
flagVer=2,
doBand=1,
minSumCC=parms["minFluxASC"],
refAnt=goodCal["bestRef"],
solInt=parms["solAInt"],
smoTimeA=2.0,
smoTimeP=1. / 60.,
doSNPlot=parms["doSNPlot"],
plotFile=plotFile,
nThreads=nThreads,
noScrat=noScrat,
logfile=logFile,
check=check,
debug=debug)
if retCode != 0:
raise RuntimeError("Error in amplitude calibration")
VLBASaveOutFiles() # Save plot file in Outfiles
# Rewrite contCals pickle file because contCals may have been updated
SaveObject(parms["contCals"], OKCalPicklefile, True)
if len(parms["contCals"]) <= 0:
logger.error(
"No calibrator sources have been detected! Stopping pipeline.")
raise RuntimeError("No calibrator sources have been detected!")
# Calibrate and average data
if parms["doCalAvg"]:
logger.info("--> Calibration and average data (doCalAvg)")
retCode = VLBACalAvg (uv, parms["avgClass"], seq, parms["CalAvgTime"], err, \
flagVer=2, doCalib=2, gainUse=0, doBand=1, BPVer=1, \
BIF=parms["CABIF"], EIF=parms["CAEIF"], \
BChan=parms["CABChan"], EChan=parms["CAEChan"], \
chAvg=parms["chAvg"], avgFreq=parms["avgFreq"], Compress=Compress, \
logfile=logFile, check=check, debug=debug)
if retCode != 0:
raise RuntimeError("Error in CalAvg")
# image targets phase only self-cal - NOTE: actually A&P
if parms["doImgTarget"] and not check:
logger.info("--> Image targets (doImgTargets)")
if not uvc:
# Get calibrated/averaged data
Aname = VLBAAIPSName(project, session)
uvname = project + "_" + session + "_" + band + "_Cal"
uvc = UV.newPAUV(uvname, Aname, parms["avgClass"], disk, seq, True,
err)
if err.isErr:
OErr.printErrMsg(err, "Error creating cal/avg AIPS data")
logger.debug("parms['targets'] = " + str(parms["targets"]))
VLBAImageCals(uv,
err,
Sources=parms["targets"],
seq=seq,
sclass=parms["outTclass"],
doCalib=2,
flagVer=2,
doBand=1,
FOV=parms["FOV"],
Robust=parms["Robust"],
maxPSCLoop=parms["maxPSCLoop"],
minFluxPSC=parms["minFluxPSC"],
solPInt=parms["solPInt"],
solMode=parms["solMode"],
maxASCLoop=parms["maxASCLoop"],
minFluxASC=parms["minFluxASC"],
solAInt=parms["solAInt"],
avgPol=parms["avgPol"],
avgIF=parms["avgIF"],
minSNR=parms["minSNR"],
refAnt=goodCal["bestRef"],
nThreads=nThreads,
noScrat=noScrat,
logfile=logFile,
check=check,
debug=debug)
# Rewrite targets pickle file because targets may have been updated
SaveObject(parms["targets"], targetsPicklefile, True)
if len(parms["targets"]) <= 0:
logger.error(
"No target sources have been detected! Stopping pipeline.")
raise RuntimeError("No target sources have been detected!")
# Phase calibration using target models
if parms["doPhaseCal"]:
logger.info("--> Phase calibration using target models (doPhaseCal)")
if not uvc:
# Get calibrated/averaged data
Aname = VLBAAIPSName(project, session)
uvname = project + "_" + session + "_" + band + "_Cal"
uvc = UV.newPAUV(uvname, Aname, parms["avgClass"], disk, seq, True,
err)
if err.isErr:
OErr.printErrMsg(err, "Error creating cal/avg AIPS data")
parms["targetModel"] = VLBAImageModel(parms["targets"],
parms["outTclass"], disk, seq,
err)
plotFile = project + "_" + session + "_" + band + ".PhaseCal.ps"
retCode = VLBAPhaseCal(uvc, err, calSou=parms["targets"], CalModel=parms["targetModel"], \
doCalib=-1, flagVer=0, doBand=-1, \
refAnt=goodCal["bestRef"], solInt=parms["manPCsolInt"], \
doSNPlot=parms["doSNPlot"], plotFile=plotFile, \
nThreads=nThreads, noScrat=noScrat, logfile=logFile, check=check, debug=debug)
if retCode != 0:
raise RuntimeError("Error in phase calibration")
VLBASaveOutFiles() # Save plot file in Outfiles
# Rewrite targets pickle file because targets may have been updated
SaveObject(parms["targets"], targetsPicklefile, True)
if len(parms["targets"]) <= 0:
logger.error(
"No target sources have been detected! Stopping pipeline.")
raise RuntimeError("No target sources have been detected!")
# Instrumental polarization calibration
if parms["doInstPol"]:
logger.info("--> Instrumental polarization calibration (doInstPol)")
# calibrators defaults to strong calibrator list
if not parms["instPolCal"]:
instPolCal = contCals
else:
instPolCal = parms["instPolCal"]
if not uvc:
# Get calibrated/averaged data
Aname = VLBAAIPSName(project, session)
uvname = project + "_" + session + "_" + band + "_Cal"
uvc = UV.newPAUV(uvname, Aname, parms["avgClass"], disk, seq, True,
err)
if err.isErr:
OErr.printErrMsg(err, "Error creating cal/avg AIPS data")
retCode = VLBAPolCal(uvc, instPolCal, err, \
doCalib=2, flagVer=0, doBand=-1, doSetJy=True, \
refAnt=goodCal["bestRef"], solInt=2.0, \
noScrat=noScrat, logfile=logFile, check=check, debug=debug)
if retCode != 0:
raise RuntimeError("Error in instrumental poln calibration")
# RL Phase (EVPA) calibration as BP table
if parms["doRLCal"] and parms["RLCal"]:
logger.info("--> RL phase calibration (doRLCal)")
if not uvc:
# Get calibrated/averaged data
Aname = VLBAAIPSName(project, session)
uvname = project + "_" + session + "_" + band + "_Cal"
uvc = UV.newPAUV(uvname, Aname, parms["avgClass"], disk, seq, True,
err)
if err.isErr:
OErr.printErrMsg(err, "Error creating cal/avg AIPS data")
retCode = VLBARLCal2(uvc, err, RLPCal=parms["RLCal"], \
doCalib=2, gainUse=2, flagVer=0, doBand=-1, doPol=True, \
refAnt=goodCal["bestRef"], niter=300, FOV=0.02/3600.0, \
nThreads=nThreads, noScrat=noScrat, logfile=logFile, check=check, debug=debug)
if retCode != 0:
raise RuntimeError("Error in RL phase calibration")
# image targets possible with Stokes I(QU)
if parms["doImgFullTarget"]:
logger.info("--> Image targets (doImgFullTargets)")
if not uvc:
# Get calibrated/averaged data
Aname = VLBAAIPSName(project, session)
uvname = project + "_" + session + "_" + band + "_Cal"
uvc = UV.newPAUV(uvname, Aname, parms["avgClass"], disk, seq, True,
err)
if err.isErr:
OErr.printErrMsg(err, "Error creating cal/avg AIPS data")
VLBAImageTargets(uvc,
err,
Sources=parms["targets"],
seq=seq,
sclass=parms["outIclass"],
doCalib=2,
flagVer=0,
doBand=-1,
Stokes=parms["Stokes"],
FOV=parms["FOV"],
Robust=parms["Robust"],
maxPSCLoop=2,
minFluxPSC=parms["minFluxPSC"],
solPInt=parms["solPInt"],
solMode="P",
maxASCLoop=parms["maxASCLoop"],
minFluxASC=parms["minFluxASC"],
solAInt=parms["solAInt"],
avgPol=parms["avgPol"],
avgIF=parms["avgIF"],
minSNR=parms["minSNR"],
refAnt=goodCal["bestRef"],
nTaper=parms["nTaper"],
Tapers=parms["Tapers"],
do3D=parms["do3D"],
nThreads=nThreads,
noScrat=noScrat,
logfile=logFile,
check=check,
debug=debug)
# Save UV data?
if parms["doSaveUV"] and (not check):
logger.info("--> Save UV data (doSaveUV)")
mess = "Write calibrated and averaged UV data to disk"
printMess(mess, logFile)
# Get calibrated/averaged data
if not uvc:
Aname = VLBAAIPSName(project, session)
uvname = project + "_" + session + "_" + band + "_Cal"
uvc = UV.newPAUV(uvname, Aname, parms["avgClass"], disk, seq, True,
err)
if err.isErr:
OErr.printErrMsg(err, "Error creating cal/avg AIPS data")
# Write
filename = project + session + band + "CalAvg.uvtab"
fuv = VLBAUVFITS(uvc, filename, 0, err, compress=Compress)
VLBAAddOutFile(filename, 'project', "Calibrated Averaged UV data")
# Save list of output files
VLBASaveOutFiles()
# Save UV data tables?
if parms["doSaveTab"] and (not check):
logger.info("--> Save UV data tables (doSaveTab)")
mess = "Write UV data tables to disk."
printMess(mess, logFile)
filename = project + session + band + "CalTab.uvtab"
fuv = VLBAUVFITSTab(uv, filename, 0, err)
VLBAAddOutFile(filename, 'project', "Calibrated AIPS tables")
# Save list of output files
VLBASaveOutFiles()
# Imaging results
outDisk = 0
if parms["doSaveImg"]:
logger.info("--> Save images (doSaveImg)")
mess = "Write images to disk."
printMess(mess, logFile)
# How many Stokes images
nstok = len(parms["Stokes"])
# Targets
logger.debug("parms['targets'] = " + str(parms['targets']))
for target in parms["targets"]:
if not check:
# intermediate images
oclass = parms["outTclass"]
x = Image.newPAImage("out", target, oclass, disk, seq, True,
err)
if (not x.exist):
print(target, "image not found. Skipping.")
continue
outfile = project + session + band + target + "." + oclass + ".fits"
mess = "Write Intermediate target " + outfile + " on disk " + str(
outDisk)
VLBAAddOutFile(outfile, target, 'Intermediate target image')
printMess(mess, logFile)
xf = VLBAImFITS(x, outfile, outDisk, err, fract=0.1)
# Save list of output files
VLBASaveOutFiles()
# Statistics
zz = imstat(x, err, logfile=logFile)
del x, xf
# Final images
for istok in range(0, nstok):
oclass = parms["Stokes"][istok:istok +
1] + parms["outIclass"][1:]
x = Image.newPAImage("out", target, oclass, disk, seq,
True, err)
outfile = project + session + band + target + "." + oclass + ".fits"
logger.info("Write " + outfile + " on disk " +
str(outDisk))
xf = VLBAImFITS(x, outfile, outDisk, err, fract=0.1)
VLBAAddOutFile(outfile, target, 'Image')
logger.info("Writing file " + outfile)
# Statistics
zz = imstat(x, err, logfile=logFile)
del x, xf
# Save list of output files
VLBASaveOutFiles()
# Calibrators
for target in parms["contCals"]:
if not check:
oclass = parms["outCclass"]
x = Image.newPAImage("out", target, oclass, disk, seq, True,
err)
if (not x.exist):
print(target, "image not found. Skipping.")
continue
outfile = project + session + band + target + "." + oclass + ".fits"
mess = "Write Calibrator " + outfile + " on disk " + str(
outDisk)
printMess(mess, logFile)
xf = VLBAImFITS(x, outfile, outDisk, err, fract=0.1)
VLBAAddOutFile(outfile, target, 'Calibrator Image')
# Statistics
zz = imstat(x, err, logfile=logFile)
del x, xf
# Save list of output files
VLBASaveOutFiles()
# end writing images loop
# Contour plots
if parms["doKntrPlots"]:
logger.info("--> Contour plots (doKntrPlots)")
VLBAKntrPlots(err,
imName=parms["targets"],
project=project,
session=session,
band=band,
disk=disk,
debug=debug)
# Save list of output files
VLBASaveOutFiles()
elif debug:
print("Not creating contour plots ( doKntrPlots = ",
parms["doKntrPlots"], " )")
# Source uv plane diagnostic plots
if parms["doDiagPlots"]:
logger.info("--> Diagnostic plots (doDiagPlots)")
# Get the highest number avgClass catalog file
Aname = VLBAAIPSName(project, session)
uvc = None
if not check:
uvname = project + "_" + session + "_" + band + "_Cal"
uvc = UV.newPAUV(uvname, Aname, parms["avgClass"], disk, seq, True,
err)
VLBADiagPlots( uvc, err, cleanUp=parms["doCleanup"], \
project=project, session=session, band=band, \
logfile=logFile, check=check, debug=debug )
# Save list of output files
VLBASaveOutFiles()
elif debug:
print("Not creating diagnostic plots ( doDiagPlots = ",
parms["doDiagPlots"], " )")
# Save metadata
srcMetadata = None
projMetadata = None
if parms["doMetadata"]:
logger.info("--> Save metadata (doMetadata)")
if not uvc:
# Get calibrated/averaged data
Aname = VLBAAIPSName(project, session)
uvname = project + "_" + session + "_" + band + "_Cal"
uvc = UV.newPAUV(uvname, Aname, parms["avgClass"], disk, seq, True,
err)
if err.isErr:
OErr.printErrMsg(err, "Error creating cal/avg AIPS data")
# Get source metadata; save to pickle file
srcMetadata = VLBASrcMetadata(uvc,
err,
Sources=parms["targets"],
seq=seq,
sclass=parms["outIclass"],
Stokes=parms["Stokes"],
logfile=logFile,
check=check,
debug=debug)
picklefile = project + "_" + session + "_" + band + "SrcReport.pickle"
SaveObject(srcMetadata, picklefile, True)
VLBAAddOutFile(picklefile, 'project', 'All source metadata')
# Get project metadata; save to pickle file
projMetadata = VLBAProjMetadata(uvc,
AIPS_VERSION,
err,
contCals=parms["contCals"],
goodCal=goodCal,
project=project,
session=session,
band=band,
dataInUVF=dataInUVF,
archFileID=archFileID)
picklefile = project + "_" + session + "_" + band + "ProjReport.pickle"
SaveObject(projMetadata, picklefile, True)
VLBAAddOutFile(picklefile, 'project', 'Project metadata')
# Write report
if parms["doHTML"]:
logger.info("--> Write HTML report (doHTML)")
VLBAHTMLReport( projMetadata, srcMetadata, \
outfile=project+"_"+session+"_"+band+"report.html", \
logFile=logFile )
# Write VOTable
if parms["doVOTable"]:
logger.info("--> Write VOTable (doVOTable)")
VLBAAddOutFile('VOTable.xml', 'project', 'VOTable report')
VLBAWriteVOTable(projMetadata, srcMetadata, filename='VOTable.xml')
# Save list of output files
VLBASaveOutFiles()
# Cleanup - delete AIPS files
if parms["doCleanup"] and (not check):
logger.info("--> Clean up (doCleanup)")
# Delete target images
# How many Stokes images
nstok = len(parms["Stokes"])
for istok in range(0, nstok):
oclass = parms["Stokes"][istok:istok + 1] + parms["outIclass"][1:]
AllDest(err, disk=disk, Aseq=seq, Aclass=oclass)
# delete Calibrator images
AllDest(err, disk=disk, Aseq=seq, Aclass=parms["outCclass"])
# Delete intermediate target images
AllDest(err, disk=disk, Aseq=seq, Aclass=parms["outTclass"])
OErr.printErrMsg(err, "Deleting AIPS images")
# Delete UV data
uv.Zap(err)
# Zap calibrated/averaged data
if not uvc:
Aname = VLBAAIPSName(project, session)
uvc = UV.newPAUV("AIPS UV DATA", Aname, parms["avgClass"], disk,
seq, True, err)
if err.isErr:
OErr.printErrMsg(err, "Error creating cal/avg AIPS data")
uvc.Zap(err)
OErr.printErrMsg(err, "Writing output/cleanup")
# Shutdown
mess = "Finished project " + project
printMess(mess, logFile)
OErr.printErr(err)
OSystem.Shutdown(ObitSys)
def pipeline(aipsSetup, parmFile):
"""
ALMA Continuum pipeline.
* *aipsSetup* = AIPS setup file
* *parmFile* = pipeline input parameters file
"""
############################# Initialize OBIT ##########################################
noScrat = []
exec(open(aipsSetup).read())
ALMAAddOutFile(aipsSetup, 'project', "Obit's AIPS setup file")
############################# Default parameters ##########################################
# Initialize parameters
parms = ALMAInitContParms()
############################# Set Project Processing parameters ##################
print "parmFile", parmFile
exec(open(parmFile).read())
ALMAAddOutFile(parmFile, 'project', 'Pipeline input parameters')
# frequency/configuration dependent default parameters
ALMAInitContFQParms(parms)
# General data parameters
band = parms["band"] # Observing band
dataClass = ("UVDa" + band)[0:6] # AIPS class of raw uv data
project = parms["project"][
0:12] # Project name (12 char or less, used as AIPS Name)
session = parms["session"] # Project session code
################################## Process #####################################
fileRoot = parms["project"] + "_" + parms["session"] + "_" + parms[
"band"] # root of file name
logFile = fileRoot + ".log" # Processing log file
uv = None
uvc = None
avgClass = ("UVAv" + band)[0:6] # Averaged data AIPS class
outIClass = parms["outIClass"] # image AIPS class
outCClass = parms["outCClass"] # image AIPS class
# Load the outputs pickle jar
ALMAFetchOutFiles()
# Logging directly to logFile
OErr.PInit(err, parms["prtLv"], logFile)
OSystem.PAllowThreads(nThreads) # Allow threads in Obit/oython
retCode = 0
ALMAAddOutFile(logFile, 'project', 'Pipeline log file')
mess = "Start project "+parms["project"]+" session "+parms["session"]+\
" "+parms["band"]+" Band"+" AIPS user no. "+str(AIPS.userno)+\
", ALMA Max. baseline "+str(parms["ALMAMaxBl"])
printMess(mess, logFile)
if debug:
pydoc.ttypager = pydoc.plainpager # don't page task input displays
mess = "Using Debug mode "
printMess(mess, logFile)
if check:
mess = "Only checking script"
printMess(mess, logFile)
# Log parameters
if parms['doLogParms']:
printMess("Parameter settings", logFile)
for p in parms:
mess = " " + p + ": " + str(parms[p])
printMess(mess, logFile)
# Save parameters to pickle jar, manifest
ParmsPicklefile = project + "_" + session + "_" + band + ".Parms.pickle" # Where results saved
SaveObject(parms, ParmsPicklefile, True)
ALMAAddOutFile(ParmsPicklefile, 'project', 'Processing parameters used')
# Are we going to be doing Hanning?
if parms["doHann"]:
loadClass = parms["band"] + "Raw"
else:
loadClass = dataClass
# Load Data from Archive directory
if parms["doLoadArchive"]:
uv = ALMAUVLoadArch(parms["archRoots"], ALMAAIPSName(project, session), loadClass, disk, parms["seq"], err, \
selConfig=parms["selConfig"], selBand=parms["selBand"], selChan=parms["selChan"], \
selChBW=parms["selChBW"], selNIF=parms["selNIF"], calInt=parms["calInt"], \
logfile=logFile, Compress=parms["Compress"], check=check, debug=debug)
if uv == None and not check:
raise RuntimeError, "Cannot load " + parms["archRoots"]
# Hanning
if parms["doHann"]:
# Set uv if not done
if uv == None and not check:
uv = UV.newPAUV("AIPS UV DATA", ALMAAIPSName(project, session),
loadClass[0:6], disk, parms["seq"], True, err)
if err.isErr:
OErr.printErrMsg(err, "Error creating AIPS data")
uv = ALMAHann(uv, ALMAAIPSName(project, session), dataClass, disk, parms["seq"], err, \
doDescm=parms["doDescm"], logfile=logFile, check=check, debug=debug)
if uv == None and not check:
raise RuntimeError, "Cannot Hann data "
# Set uv if not done
if uv == None and not check:
uv = UV.newPAUV("AIPS UV DATA", ALMAAIPSName(project, session), dataClass[0:6], \
disk, parms["seq"], True, err)
if err.isErr:
OErr.printErrMsg(err, "Error creating AIPS data")
# Clear any old calibration/editing
if parms["doClearTab"]:
mess = "Clear previous calibration"
printMess(mess, logFile)
ALMAClearCal(uv,
err,
doGain=parms["doClearGain"],
doFlag=parms["doClearFlag"],
doBP=parms["doClearBP"],
check=check)
OErr.printErrMsg(err, "Error resetting calibration")
# Copy FG 1 to FG 2
if parms["doCopyFG"]:
mess = "Copy FG 1 to FG 2"
printMess(mess, logFile)
retCode = ALMACopyFG(uv,
err,
logfile=logFile,
check=check,
debug=debug)
if retCode != 0:
raise RuntimeError, "Error Copying FG table"
# Drop end channels of spectra? Only if new FG 2
if parms["doCopyFG"] and (parms["BChDrop"] > 0) or (parms["EChDrop"] > 0):
# Channels based on original number, reduced if Hanning
nchan = uv.Desc.Dict["inaxes"][uv.Desc.Dict["jlocf"]]
fact = max(1, parms["selChan"] / nchan) # Hanning reduction factor
BChDrop = parms["BChDrop"] / fact
EChDrop = parms["EChDrop"] / fact
mess = "Trim %d channels from start and %d from end of each spectrum" % (
BChDrop, EChDrop)
printMess(mess, logFile)
retCode = ALMADropChan (uv, BChDrop, EChDrop, err, flagVer=parms["editFG"], \
logfile=logFile, check=check, debug=debug)
if retCode != 0:
raise RuntimeError, "Error Copying FG table"
# Get list of source in data
AllSource = ALMAAllSource(uv,
err,
logfile=logFile,
check=check,
debug=debug)
# Edit source lists to remove sources not present
for c in parms["PCInsCals"]:
if c not in AllSource:
parms["PCInsCals"].remove(c)
for t in parms["targets"]:
if t not in AllSource:
parms["targets"].remove(t)
if parms["XYGainSource"] not in AllSource:
parms["XYGainSource"] = None
if parms["XYDelaySource"] not in AllSource:
parms["XYDelaySource"] = None
for c in parms["DCals"]:
if c['Source'] not in AllSource:
c['Source'] = None
for c in parms["BPCals"]:
if c['Source'] not in AllSource:
c['Source'] = None
for c in parms["PCals"]:
if c['Source'] not in AllSource:
c['Source'] = None
for c in parms["ACals"]:
if c['Source'] not in AllSource:
c['Source'] = None
# Special editing
if parms["doEditList"] and not check:
mess = "Special editing"
printMess(mess, logFile)
for edt in parms["editList"]:
UV.PFlag(uv,err,timeRange=[dhms2day(edt["timer"][0]),dhms2day(edt["timer"][1])], \
flagVer=parms["editFG"], Ants=edt["Ant"], Chans=edt["Chans"], IFs=edt["IFs"], \
Stokes=edt["Stokes"], Reason=edt["Reason"])
OErr.printErrMsg(err, "Error Flagging")
# Quack to remove data from start and end of each scan
if parms["doQuack"]:
retCode = ALMAQuack (uv, err, begDrop=parms["quackBegDrop"], endDrop=parms["quackEndDrop"], \
Reason=parms["quackReason"], \
logfile=logFile, check=check, debug=debug)
if retCode != 0:
raise RuntimeError, "Error Quacking data"
# Flag antennas shadowed by others?
if parms["doShad"]:
retCode = ALMAShadow (uv, err, shadBl=parms["shadBl"], \
logfile=logFile, check=check, debug=debug)
if retCode != 0:
raise RuntimeError, "Error Shadow flagging data"
# Apply online calibration
if parms["doOnlineCal"]:
retCode = ALMAOnlineCal(uv,
err,
logfile=logFile,
check=check,
debug=debug)
if retCode != 0:
raise RuntimeError, "Error applying online calibration"
# Median window time editing, for RFI impulsive in time
if parms["doMedn"]:
mess = "Median window time editing, for RFI impulsive in time:"
printMess(mess, logFile)
retCode = ALMAMedianFlag (uv, " ", err, noScrat=noScrat, nThreads=nThreads, \
avgTime=parms["avgTime"], avgFreq=parms["avgFreq"], chAvg= parms["chAvg"], \
timeWind=parms["timeWind"], flagVer=2,flagSig=parms["mednSigma"], \
logfile=logFile, check=check, debug=False)
if retCode != 0:
raise RuntimeError, "Error in MednFlag"
# Median window frequency editing, for RFI impulsive in frequency
if parms["doFD1"]:
mess = "Median window frequency editing, for RFI impulsive in frequency:"
printMess(mess, logFile)
retCode = ALMAAutoFlag (uv, " ", err, flagVer=2, doCalib=-1, doBand=-1, \
timeAvg=parms["FD1TimeAvg"], \
doFD=True, FDmaxAmp=1.0e20, FDmaxV=1.0e20, FDwidMW=parms["FD1widMW"], \
FDmaxRMS=[1.0e20,0.1], FDmaxRes=parms["FD1maxRes"], \
FDmaxResBL= parms["FD1maxRes"], \
nThreads=nThreads, logfile=logFile, check=check, debug=debug)
if retCode != 0:
raise RuntimeError, "Error in AutoFlag"
# RMS/Mean editing for calibrators
if parms["doRMSAvg"]:
mess = "RMS/Mean editing for calibrators:"
printMess(mess, logFile)
clist = ALMACombineCals(parms["ACals"], parms["PCals"],
parms["DCals"]) # Calibrator list
retCode = ALMAAutoFlag (uv, clist, err, flagVer=2, doCalib=-1, doBand=-1, \
RMSAvg=parms["RMSAvg"], timeAvg=parms["RMSTimeAvg"], \
nThreads=nThreads, logfile=logFile, check=check, debug=debug)
if retCode != 0:
raise RuntimeError, "Error in AutoFlag"
# Need to find a reference antenna? See if we have saved it?
if (parms["refAnt"] <= 0):
refAnt = FetchObject(project + "_" + session + "_" + band +
".refAnt.pickle")
if refAnt:
parms["refAnt"] = refAnt
# Use bandpass calibrator and center half of each spectrum
if parms["refAnt"] <= 0:
mess = "Find best reference antenna: run Calib on BP Cal(s) "
printMess(mess, logFile)
parms["refAnt"] = ALMAGetRefAnt(uv, parms["BPCals"], err, flagVer=2, \
solInt=parms["bpsolint1"], nThreads=nThreads, \
logfile=logFile, check=check, debug=debug)
if err.isErr:
raise RuntimeError, "Error finding reference antenna"
if parms["refAnts"][0] <= 0:
parms["refAnts"][0] = parms["refAnt"]
mess = "Picked reference antenna " + str(parms["refAnt"])
printMess(mess, logFile)
# Save it
ParmsPicklefile = project + "_" + session + "_" + band + ".Parms.pickle" # Where results saved
SaveObject(parms, ParmsPicklefile, True)
refAntPicklefile = project + "_" + session + "_" + band + ".refAnt.pickle" # Where results saved
SaveObject(parms["refAnt"], refAntPicklefile, True)
# Plot Raw, edited data?
if parms["doRawSpecPlot"] and parms["plotSource"]:
mess = "Raw Spectral plot for: " + parms["plotSource"]
printMess(mess, logFile)
plotFile = "./" + fileRoot + "RawSpec.ps"
retCode = ALMASpectrum(uv, parms["plotSource"], parms["plotTime"], plotFile, parms["refAnt"], err, \
flagVer=2, Stokes=["XX","YY"], doband=-1, \
check=check, debug=debug, logfile=logFile )
if retCode != 0:
raise RuntimeError, "Error in Plotting spectrum"
ALMAAddOutFile(plotFile, 'project', 'Pipeline log file')
# delay calibration
if parms["doDelayCal"] and parms["DCals"] and not check:
plotFile = "./" + fileRoot + "DelayCal.ps"
retCode = ALMADelayCal(uv, parms["DCals"], err, \
BChan=parms["delayBChan"], EChan=parms["delayEChan"], \
doCalib=2, flagVer=2, doBand=-1, \
solInt=parms["delaySolInt"], smoTime=parms["delaySmoo"], \
refAnts=[parms["refAnt"]], doTwo=parms["doTwo"],
doZeroPhs=parms["delayZeroPhs"], \
doPlot=parms["doSNPlot"], plotFile=plotFile, \
nThreads=nThreads, noScrat=noScrat, \
logfile=logFile, check=check, debug=debug)
if retCode != 0:
raise RuntimeError, "Error in delay calibration"
# Plot corrected data?
if parms["doSpecPlot"] and parms["plotSource"]:
plotFile = "./" + fileRoot + "DelaySpec.ps"
retCode = ALMASpectrum(uv, parms["plotSource"], parms["plotTime"], \
plotFile, parms["refAnt"], err, \
flagVer=2, Stokes=["XX","YY"], doband=-1, \
check=check, debug=debug, logfile=logFile )
if retCode != 0:
raise RuntimeError, "Error in Plotting spectrum"
# Bandpass calibration
if parms["doBPCal"] and parms["BPCals"]:
plotFile = "./" + fileRoot + "BPCal.ps"
retCode = ALMABPCal(uv, parms["BPCals"], err, noScrat=noScrat, solInt1=parms["bpsolint1"], \
solInt2=parms["bpsolint2"], solMode=parms["bpsolMode"], \
BChan1=parms["bpBChan1"], EChan1=parms["bpEChan1"], \
BChan2=parms["bpBChan2"], EChan2=parms["bpEChan2"], ChWid2=parms["bpChWid2"], \
doCenter1=parms["bpDoCenter1"], refAnt=parms["refAnt"], \
UVRange=parms["bpUVRange"], doCalib=2, gainUse=0, flagVer=2, doPlot=False, \
doAmpEdit=parms["doAmpEdit"], ampSigma=parms["ampSigma"], \
doBPPlot=parms["doBPPlot"], plotBPFile=plotFile, \
nThreads=nThreads, logfile=logFile, check=check, debug=debug)
if retCode != 0:
raise RuntimeError, "Error in Bandpass calibration"
# Plot corrected data?
if parms["doSpecPlot"] and parms["plotSource"]:
plotFile = "./" + fileRoot + "BPSpec.ps"
retCode = ALMASpectrum(uv, parms["plotSource"], parms["plotTime"], plotFile, \
parms["refAnt"], err, flagVer=2, Stokes=["XX","YY"], doband=1, \
check=check, debug=debug, logfile=logFile )
if retCode != 0:
raise RuntimeError, "Error in Plotting spectrum"
# set X/Y gains and initial calibration
if parms["doXYFixGain"] and parms["XYGainSource"] and not check:
mess = "Fix X/Y gain ratios"
printMess(mess, logFile)
retCode = ALMAXYGain(uv, err, \
XYCal=parms["XYGainSource"],timerange=parms["XYGainTime"], \
doCalib=2, gainUse=0, doBand=1, flagVer=2, refAnt=parms["refAnt"], \
nThreads=nThreads, noScrat=noScrat, logfile=logFile, \
check=check, debug=debug)
if retCode != 0:
raise RuntimeError, "Error in X-Y gain fix"
# Self calibrate calibrators
if parms["doImgCal"] and not check:
mess = "SelfCalibrate/Image calibrators"
printMess(mess, logFile)
src = ALMACombineCals(parms["ACals"], parms["PCals"],
parms["APCals"]) # Source list
ALMAImageCals(uv, err, Sources=src, seq=parms["seq"], \
sclass=parms["outCClass"], doCalib=2, flagVer=2, doBand=1, FOV=parms["CalFOV"], \
maxPSCLoop=parms["maxPSCLoop"], minFluxPSC=parms["minFluxPSC"], solPInt=parms["solPInt"], \
maxASCLoop=parms["maxASCLoop"], minFluxASC=parms["minFluxASC"],\
solAInt=parms["solAInt"], avgPol=parms["avgPol"], Niter=1000, \
avgIF=parms["avgIF"], minSNR=parms["minSNR"],\
refAnt=parms["refAnt"], nThreads=nThreads, noScrat=noScrat,\
logfile=logFile, check=check, debug=debug)
# Self calibrated models now available
ALMAImageModel(parms["ACals"], parms["outCClass"], disk, parms["seq"], err)
ALMAImageModel(parms["PCals"], parms["outCClass"], disk, parms["seq"], err)
ALMAImageModel(parms["APCals"], parms["outCClass"], disk, parms["seq"],
err)
# Phase Calibrate
if parms["doPhaseCal"]:
plotFile = "./" + fileRoot + "PhaseCal.ps"
retCode = ALMAPhaseCal (uv, parms["PCals"], err, ACals=parms["ACals"], \
doCalib=2, doBand=1, BPVer=1, flagVer=2, refAnt=parms["refAnt"], \
BChan=parms["ampBChan"], EChan=parms["ampEChan"], \
solInt=parms["solPInt"], avgIF = parms['avgIF'], \
ampScalar=parms["ampScalar"], doPlot=parms["doSNPlot"], plotFile=plotFile, \
nThreads=nThreads, noScrat=noScrat, logfile=logFile, check=check, debug=debug)
if retCode != 0:
raise RuntimeError, "Error calibrating"
# Amp & phase Calibrate
if parms["doAmpPhaseCal"]:
plotFile = "./" + fileRoot + "APCal.ps"
retCode = ALMACalAP (uv, [], parms["ACals"], err, PCals=parms["APCals"], \
doCalib=2, doBand=1, BPVer=1, flagVer=2, \
BChan=parms["ampBChan"], EChan=parms["ampEChan"], \
solInt=parms["solAInt"], solSmo=parms["solSmo"], ampScalar=parms["ampScalar"], \
doAmpEdit=parms["doAmpEdit"], ampSigma=parms["ampSigma"], \
ampEditFG=parms["ampEditFG"], \
doPlot=parms["doSNPlot"], plotFile=plotFile, refAnt=parms["refAnt"], \
nThreads=nThreads, noScrat=noScrat, logfile=logFile, check=check, debug=debug)
if retCode != 0:
raise RuntimeError, "Error calibrating"
# More editing
if parms["doAutoFlag"]:
mess = "Post calibration editing:"
printMess(mess, logFile)
clist = []
retCode = ALMAAutoFlag (uv, clist, err, flagVer=2, \
doCalib=2, gainUse=0, doBand=1, BPVer=1, \
IClip=parms["IClip"], minAmp=parms["minAmp"], timeAvg=parms["timeAvg"], \
doFD=parms["doAFFD"], FDmaxAmp=parms["FDmaxAmp"], FDmaxV=parms["FDmaxV"], \
FDwidMW=parms["FDwidMW"], FDmaxRMS=parms["FDmaxRMS"], \
FDmaxRes=parms["FDmaxRes"], FDmaxResBL=parms["FDmaxResBL"], \
FDbaseSel=parms["FDbaseSel"], \
nThreads=nThreads, logfile=logFile, check=check, debug=debug)
if retCode != 0:
raise RuntimeError, "Error in AutoFlag"
# Calibrate and average data
if parms["doCalAvg"]:
retCode = ALMACalAvg (uv, avgClass, parms["seq"], parms["CalAvgTime"], err, \
flagVer=2, doCalib=2, gainUse=0, doBand=1, BPVer=1, doPol=False, \
avgFreq=parms["CAavgFreq"], chAvg=parms["CAchAvg"], \
BChan=parms["CABChan"], EChan=parms["CAEChan"], \
BIF=parms["CABIF"], EIF=parms["CAEIF"], Compress=parms["Compress"], \
nThreads=nThreads, logfile=logFile, check=check, debug=debug)
if retCode != 0:
raise RuntimeError, "Error in CalAvg"
# Get calibrated/averaged data
if not check:
try:
uv = UV.newPAUV("AIPS UV DATA", ALMAAIPSName(project, session), avgClass[0:6], \
disk, parms["seq"], True, err)
except Exception, exception:
mess = "No Averaged/calibrated datafile"
printMess(mess, logFile)
return
usage = "%prog [options] uvfitsfile"
description = "Image a uvfits dataset"
parser = OptionParser(usage=usage, description=description)
parser.add_option("--targets", default=None, help="List of targets to image")
parser.add_option("--refant", type='int', default=0, help="Ref ant to use")
parser.add_option("--blavg",
action="store_true",
help="BL dep. avg. and Stokes I?")
parser.add_option("--scratch", help="Location of aips disk to create")
(options, args) = parser.parse_args()
filebase = os.path.basename(os.path.splitext(args[0])[0])
# Obit error logging
err = OErr.OErr()
OErr.PInit(err, 2, '/dev/null')
ObitSys = AIPSSetup.AIPSSetup(err, scratchdir=options.scratch)
# Get the set up AIPS environment.
AIPS_ROOT = os.environ['AIPS_ROOT']
AIPS_VERSION = os.environ['AIPS_VERSION']
user = OSystem.PGetAIPSuser()
AIPS.userno = user
disk = 1
fitsdisk = 1
nam = filebase
cls = "Raw"
seq = 1
logFile = "Image.log" # Processing log file
#astertemplate=ObitTalkUtil.FITSDir.FITSdisks[fitsdisk]+'MKATTemplate.uvtab.gz'