def __init__(self):
"""
Constructor
Largely derived from
https://github.com/bill-cotton/Obit/blob/master/ObitSystem/Obit/share/scripts/AIPSSetup.py
"""
# Get the current configuration
cfg = kc.get_config()
self.err = err = OErr.OErr()
self.obitsys = OSystem.OSystem("Pipeline", 1, cfg['userno'], 0, [" "],
0, [" "], True, False, err)
OErr.printErrMsg(err, "Error starting Obit System")
# Setup AIPS userno
AIPS.userno = cfg['userno']
# Setup Obit Environment
ObitTalkUtil.SetEnviron(AIPS_ROOT=cfg['aipsroot'],
AIPS_VERSION=cfg['aipsversion'],
OBIT_EXEC=cfg['obitexec'],
DA00=cfg['da00'],
ARCH="LINUX",
aipsdirs=cfg['aipsdirs'],
fitsdirs=cfg['fitsdirs'])
class AIPS:
"""Container for several AIPS-related default values."""
# Default AIPS user ID.
userno = 0
# List of available proxies.
proxies = [LocalProxy]
# AIPS disk mapping.
disks = [None] # Disk numbers are one-based.
# Check for disks already in the system
import OErr, Obit
err = OErr.OErr()
numb = Obit.AIPSGetNumDisk(err.me)
disk = 0
for i in range(0, numb):
disk += 1
disks.append(AIPSDisk(None, disk, Obit.AIPSGetDirname(disk, err.me)))
# AIPS seems to support a maximum of 35 disks.
for i in range(1, 35 - numb):
disk += 1
area = 'DA' + ehex(disk, 2, '0')
dirname = os.getenv(area)
if not area in os.environ:
break
disks.append(AIPSDisk(None, disk, dirname))
continue
# Message log.
log = None
# Debug log.
debuglog = None
def _verify(self, desc):
# Initialize Obit if needed
if not OSystem.PIsInit() and desc:
AIPSdirs = desc["dirs"]
userno = desc["userno"]
popsno = 1
if not self.err:
self.err=OErr.OErr()
self.ObitSys=OSystem.OSystem ("", popsno, userno, \
len(AIPSdirs), AIPSdirs, \
0, [], True, False, self.err)
OErr.printErrMsg(self.err, "Error with Obit startup")
self.doInit = True
else:
self.doInit = False
try:
data = self._init(desc)
except OErr.OErr as err:
print(err)
#OErr.printErrMsg(err, "AIPSData._verify")
if self.doInit: # Initialized Obit?
OSystem.Shutdown(self.ObitSys)
raise RuntimeError("Cannot open data set %s" % desc['name'])
#OErr.printErrMsg(self.err, "AIPSData._verify")
return data
def __init__(self, name, klass, disk, seq):
self._err = OErr.OErr()
OSystem.PSetAIPSuser(AIPS.userno)
self._data = UV.newPAUV(name, name, klass, disk, seq, True, self._err)
if self._err.isErr:
raise RuntimeError
return
def _verify(self, desc):
# Initialize Obit if needed
if not self.err:
self.err=OErr.OErr()
try:
data = self._init(desc)
except OErr.OErr, err:
print err
#OErr.printErrMsg(err, "FITSData._verify")
raise RuntimeError, "Cannot open data set %s" % desc['filename']
def _disk_list_add(*args):
"""Add DAxx entries for the directories supplied and update NVOL."""
try:
nvol = int(os.environ['NVOL'])
except KeyError:
nvol = 0
for i in range(len(args)):
os.environ['DA%s' % ehex(nvol + 1 + i, 2, 0)] = args[i]
AIPS.disks.append(AIPS.AIPSDisk(None, nvol + 1 + i))
err = OErr.OErr()
Obit.AIPSSetDirname(nvol + 1 + i, args[i] + '/', err.me)
if err.isErr:
raise RuntimeError
os.environ['NVOL'] = str(nvol + len(args))
def cat(self, disk, userno, url, AIPSdirs):
# Init Obit if needed
if not OSystem.PIsInit():
popsno = 1
if not self.err:
self.err=OErr.OErr()
ObitSys=OSystem.OSystem ("", popsno, userno, \
len(AIPSdirs), AIPSdirs, \
0, [], True, False, self.err)
OErr.printErrMsg(self.err, "Error with Obit startup")
doInit = True
else:
doInit = False
_userno = OSystem.PGetAIPSuser()
OSystem.PSetAIPSuser(userno)
try:
num_slots = AIPSDir.PNumber(disk, userno, self.err)
except OErr.OErr as err:
OErr.PClear(err)
if doInit: # Initialized Obit?
OSystem.Shutdown(ObitSys)
self.doInit = False
return []
catalog = []
for slot in range(1, num_slots):
entry = AIPSDir.PInfo(disk, userno, slot, self.err)
if entry:
catalog.append((slot, entry))
pass
continue
# Restore Obit to initial state
OSystem.PSetAIPSuser(_userno)
if doInit: # Initialized Obit?
OSystem.Shutdown(ObitSys)
self.doInit = False
OErr.printErrMsg(self.err, "Error with AIPS Catalog")
return catalog
def __init__(self, data, name, version):
self._err = OErr.OErr()
self._table = data.NewTable(3, 'AIPS ' + name, version, self._err)
self._table.Open(3, self._err)
if self._err.isErr:
raise self._err
header = self._table.Desc.Dict
self._columns = {}
self._keys = []
for column in header['FieldName']:
# Convert the AIPS ccolumn names into acceptable Python
# identifiers.
key = column.lower()
key = key.replace(' ', '_')
key = key.rstrip('.')
key = key.replace('.', '_')
self._columns[key] = column
self._keys.append(key)
continue
self.name = name
self.version = header['version']
return
def pixinterpold(pixx, pixy, imagedata):
err = OErr.OErr()
image = makeobitimage(imagedata, err)
image.Open(Image.READONLY, err)
image.Read(err)
pixels = image.FArray
values = numpy.zeros((25, ))
points = numpy.zeros((
25,
2,
))
nloop = -1
for x in range(int(pixx - 3), int(pixx + 2)):
for y in range(int(pixy - 3), int(pixy + 2)):
nloop += 1
values[nloop] = pixels.get(x, y)
points[nloop] = [x, y]
val = interpolate.griddata(points,
values, [pixx - 1, pixy - 1],
method='cubic')
return val[0]
def fitsout(inimagedata, outfile):
err = OErr.OErr()
x = Image.newPAImage(inimagedata[0], inimagedata[0], inimagedata[1],
inimagedata[2], inimagedata[3], True, err)
#inimage=makeaipsimage(inimagedata)
if os.path.exists(outfile):
print 'Cannot create ' + outfile
raise IOError
xf = EVLAImFITS(x, outfile, 0, err)
#fittp = AIPSTask('fittp')
#fittp.indata = inimage
#fittp.dataout = outfile
#fittp.go()
# Correct an AIPS bug with NaN pixels.
outimage = pyfits.open(outfile, mode='update')
#print numpy.nanmin(outimage[0].data)
outimage[0].header.set('datamin', numpy.nanmin(outimage[0].data))
outimage[0].header.set('datamax', numpy.nanmax(outimage[0].data))
outimage.flush()
outimage.close()
def cat(self, disk, userno, url, AIPSdirs):
# Init Obit if needed
if not OSystem.PIsInit():
popsno = 1
if not self.err:
self.err=OErr.OErr()
ObitSys=OSystem.OSystem ("", popsno, userno, \
len(AIPSdirs), AIPSdirs, \
0, [], True, False, self.err)
OErr.printErrMsg(self.err, "Error with Obit startup")
doInit = True
else:
doInit = False
_userno = OSystem.PGetAIPSuser()
OSystem.PSetAIPSuser(userno)
try:
num_slots = AIPSDir.PNumber(disk, userno, self.err)
except OErr.OErr, err:
OErr.PClear(err)
if doInit: # Initialized Obit?
OSystem.Shutdown(ObitSys)
self.doInit = False
return []
# python/Obit equivalent of AIPSish 1.0/PBCor
# Arguments:
# 1) Name of input image
# 2) name of image from which pointing is to be obtained
# 3) Name of the output image
# example
# Python scriptPBApply.py ComaAGBT.fits ComaA-P-Bsub2.fits ComaAGBTPBcor.fits
import sys, Obit, Image, ImageUtil, OSystem, OErr
# Init Obit
err = OErr.OErr()
ObitSys = OSystem.OSystem("PBApply", 1, 100, 1, ["../AIPSdata/"], 1,
["../PythonData/"], 1, 0, err)
OErr.printErrMsg(err, "Error with Obit startup")
# For debugging
#print sys.argv
#Obit.Bomb()
# Get file names
inFile = sys.argv[1]
pntFile = sys.argv[2]
outFile = sys.argv[3]
inDisk = 1
outDisk = 1
# Set data
inImage = Image.newPImage("Input image", inFile, inDisk, 1, err)
pntImage = Image.newPImage("Pointing image", pntFile, inDisk, 1, err)
outImage = Image.newPImage("Output image", outFile, outDisk, 0, err)
def MKContPipeline(files, outputdir, **kwargs):
"""MeerKAT Continuum pipeline.
Parameters
----------
files : list
h5 filenames (note: support for multiple h5 files
i.e. ConcatenatedDataSet is not currently supported)
outputdir : string
Directory location to write output data,
scratchdir : string, optional
The directory location of the aips disk
parmFile : string, optional
Overwrite the default imaging parameters using this parameter file.
"""
if len(files) == 1:
h5file = files[0]
else:
h5file = files
############### Initialize katfile object #########################
OK = False
# Open the h5 file as a katfile object
try:
#open katfile and perform selection according to kwargs
katdal_ref_ant = kwargs.get('katdal_refant', '')
katdal_retries = kwargs.get('katdal_retries', 2)
katdal_timeout = kwargs.get('katdal_timeout', 300)
katdata = katfile.open(h5file,
ref_ant=katdal_ref_ant,
timeout=katdal_timeout,
retries=katdal_retries)
OK = True
except Exception as exception:
print(exception)
if not OK:
raise KATUnimageableError("Unable to read MVF data in " + str(h5file))
# If we are doing polcal- search for the most recent delaycal observation
if kwargs.get('polcal'):
if kwargs.get('delaycal_mvf') is None:
# Automatically determine delay_cal CBID
delay_katdata = KATGetDelayCal(h5file,
katdata,
timeout=katdal_timeout,
retries=katdal_retries)
else:
# Use the user supplied one
delay_katdata = KATGetDelayCal(kwargs.get('delaycal_mvf'))
kwargs["delay_katdata"] = delay_katdata
# Die gracefully if we cannot write to the output area...
if not os.path.exists(outputdir):
print('Specified output directory: ' + outputdir + 'does not exist.')
exit(-1)
# Obit error logging
err = OErr.OErr()
#################### Initialize filenames #######################################################
nameRoot = katdata.obs_params.get('capture_block_id',
katdata.experiment_id)
if type(nameRoot) == list:
nameRoot = nameRoot[0]
fileRoot = os.path.join(outputdir, nameRoot) # root of file name
logFile = fileRoot + ".log" # Processing log file
avgClass = ("UVAv")[0:6] # Averaged data AIPS class
manifestfile = outputdir + '/manifest.pickle'
############################# Initialize OBIT and AIPS ##########################################
noScrat = []
# Logging directly to logFile
OErr.PInit(err, 2, logFile)
EVLAAddOutFile(os.path.basename(logFile), 'project', 'Pipeline log file')
if kwargs.get('reuse'):
ObitSys = AIPSSetup.AIPSSetup(err,
configfile=kwargs.get('configFile'),
scratchdir=kwargs.get('scratchdir'),
aipsdisk=kwargs.get('aipsdisk'),
overwrite=False)
else:
ObitSys = AIPSSetup.AIPSSetup(err,
configfile=kwargs.get('configFile'),
scratchdir=kwargs.get('scratchdir'),
aipsdisk=kwargs.get('aipsdisk'))
# Get the set up AIPS environment.
AIPS_ROOT = os.environ['AIPS_ROOT']
AIPS_VERSION = os.environ['AIPS_VERSION']
nThreads = 72
user = OSystem.PGetAIPSuser()
AIPS.userno = user
disk = 1
fitsdisk = 1
nam = nameRoot[:10]
clss = "Raw"
seq = 1
############### Condition data #########################
#Get calibrator models
fluxcals = katpoint.Catalogue(
open(FITSDir.FITSdisks[1] + "/PERLEY_BUTLER_2013.csv"))
#Condition data (get bpcals, update names for aips conventions etc)
KATh5Condition(katdata, fluxcals, err)
############################# Initialise Parameters ##########################################
####### Initialize parameters dictionary #####
parms = KATInitContParms()
parms['PolCal'] = kwargs.get('polcal')
parms['XYtarg'] = kwargs.get('XYtarg')
# Get default XYtarg if it is not set
targs = [targ.name for targ in katdata.catalogue.targets]
if parms['PolCal']:
if parms['XYtarg'] is None:
GOTTARG = False
for targ in ['1934-638', '0408-65']:
if targ in targs:
parms['XYtarg'] = targ
GOTTARG = True
break
if not GOTTARG:
raise RuntimeError(
'No default targets (1934-638, 0408-65) for XYFix. Cannot run in PolCal mode.'
)
else:
if parms['XYtarg'] not in targs:
raise RuntimeError(
'XYtarg target %s not in observation. Cannot run in PolCal mode.'
% (parms['XYtarg']))
####### User defined parameters ######
if kwargs.get('parmFile'):
print("parmFile", kwargs.get('parmFile'))
exec(open(kwargs.get('parmFile')).read())
EVLAAddOutFile(os.path.basename(kwargs.get('parmFile')), 'project',
'Pipeline input parameters')
###################### Data selection and static edits ############################################
# Select data based on static imageable parameters
KATh5Select(katdata, parms, err, **kwargs)
# General AIPS data parameters at script level
dataClass = ("UVDa")[0:6] # AIPS class of raw uv data
delayClass = "DELA"
band = katdata.spectral_windows[0].product #Correlator product
project = os.path.basename(os.path.splitext(files[0])[0])[
0:10] # Project name (12 char or less, used as AIPS Name)
outIClass = parms["outIClass"] # image AIPS class
debug = parms["debug"]
check = parms["check"]
####################### Import data into AIPS #####################################################
# Reuse or nay?
sw = katdata.spectral_windows[katdata.spw]
# Pick up static flags
if sw.band == 'L':
sflags = FetchObject(ObitTalkUtil.FITSDir.FITSdisks[fitsdisk] +
'maskred.pickle')
if kwargs.get('flag', None):
mess = 'Using static RFI mask in file %s for L-band' % (
ObitTalkUtil.FITSDir.FITSdisks[fitsdisk] + 'maskred.pickle', )
printMess(mess, logFile)
elif sw.band == 'UHF':
sflags = FetchObject(ObitTalkUtil.FITSDir.FITSdisks[fitsdisk] +
'maskredUHF.pickle')
if kwargs.get('flag', None):
mess = 'Using static RFI mask in file %s for UHF-band' % (
ObitTalkUtil.FITSDir.FITSdisks[fitsdisk] +
'maskredUHF.pickle', )
printMess(mess, logFile)
else:
sflags = np.zeros(sw.num_chans, dtype=np.bool)
sflags = sflags[katdata.channels]
# Construct a template uvfits file from master template
mastertemplate = ObitTalkUtil.FITSDir.FITSdisks[
fitsdisk] + 'MKATTemplate.uvtab.gz'
outtemplate = nam + '.uvtemp'
if kwargs.get('reuse'):
uv = UV.newPAUV("AIPS UV DATA", EVLAAIPSName(project), dataClass, disk,
seq, True, err)
obsdata = KATH5toAIPS.GetKATMeta(katdata, err)
# Extract AIPS parameters of the uv data to the metadata
obsdata["Aproject"] = uv.Aname
obsdata["Aclass"] = uv.Aclass
obsdata["Aseq"] = uv.Aseq
obsdata["Adisk"] = disk
obsdata["calInt"] = katdata.dump_period
obsdata["fitsdisk"] = fitsdisk
# TODO: Check if the input data has been Hanned.
doneHann = True
else:
mess = '\nLoading UV data with CBID: %s' % (
katdata.obs_params['capture_block_id'], )
printMess(mess, logFile)
KATH5toAIPS.MakeTemplate(mastertemplate, outtemplate, katdata)
uv = OTObit.uvlod(outtemplate, 0, EVLAAIPSName(project), clss, disk,
seq, err)
obsdata = KATH5toAIPS.KAT2AIPS(katdata,
uv,
disk,
fitsdisk,
err,
calInt=katdata.dump_period,
static=sflags,
**kwargs)
MakeIFs.UVMakeIF(uv, 8, err, solInt=katdata.dump_period)
os.remove(outtemplate)
if parms["PolCal"]:
mess = '\nLoading delay calibration with CBID: %s' % (
delay_katdata.obs_params['capture_block_id'], )
printMess(mess, logFile)
# Load the delay cal observation
KATH5toAIPS.MakeTemplate(mastertemplate, outtemplate, katdata)
delay_uv = OTObit.uvlod(outtemplate, 0, EVLAAIPSName(project),
delayClass, disk, seq, err)
KATH5toAIPS.KAT2AIPS(delay_katdata,
delay_uv,
disk,
fitsdisk,
err,
calInt=katdata.dump_period,
static=sflags,
flag=False)
MakeIFs.UVMakeIF(delay_uv, 8, err, solInt=katdata.dump_period)
os.remove(outtemplate)
# Print the uv data header to screen.
uv.Header(err)
############################# Set Project Processing parameters ###################################
# Parameters derived from obsdata and katdata
KATGetObsParms(obsdata, katdata, parms, logFile)
###### Initialise target parameters #####
KATInitTargParms(katdata, parms, err)
# Load the outputs pickle jar
EVLAFetchOutFiles()
OSystem.PAllowThreads(nThreads) # Allow threads in Obit/oython
retCode = 0
doBand = -1
BPVer = 0
maxgap = max(parms["CalAvgTime"], 160. * katdata.dump_period) / 60.
################### Start processing ###############################################################
mess = "Start project "+parms["project"]+" AIPS user no. "+str(AIPS.userno)+\
", KAT7 configuration "+parms["KAT7Cfg"]
printMess(mess, logFile)
if debug:
pydoc.ttypager = pydoc.plainpager # don't page task input displays
mess = "Using Debug mode "
printMess(mess, logFile)
if check:
mess = "Only checking script"
printMess(mess, logFile)
# Log parameters
printMess("Parameter settings", logFile)
for p in parms:
mess = " " + p + ": " + str(parms[p])
printMess(mess, logFile)
clist = []
for DCal in parms["DCals"]:
if DCal["Source"] not in clist:
clist.append(DCal["Source"])
for PCal in parms["PCals"]:
if PCal["Source"] not in clist:
clist.append(PCal["Source"])
for ACal in parms["ACals"]:
if ACal["Source"] not in clist:
clist.append(ACal["Source"])
if kwargs.get('targets') is not None:
targets = [
targ.name for targ in katdata.catalogue
if (targ.name not in clist) and (
targ.name in kwargs.get('targets').split(','))
]
else:
targets = [
targ.name for targ in katdata.catalogue if (targ.name not in clist)
]
refAnt = kwargs.get('refant')
if refAnt is not None:
try:
SaveObject(obsdata['antLookup'][refAnt],
fileRoot + ".refAnt.pickle", True)
except:
mess = "Select reference antenna " + refAnt + " not in antenna table."
printMess(mess, logFile)
print(mess)
refAnt = FetchObject(fileRoot + ".refAnt.pickle")
# Save parameters to pickle jar, manifest
ParmsPicklefile = fileRoot + ".Parms.pickle" # Where results saved
SaveObject(parms, ParmsPicklefile, True)
EVLAAddOutFile(os.path.basename(ParmsPicklefile), 'project',
'Processing parameters used')
loadClass = dataClass
# Hanning - only if not reusing
doneHann = False
if not kwargs.get('reuse'):
if parms["doHann"]:
uv = KATHann(uv, EVLAAIPSName(project), dataClass, disk, seq, err, \
doDescm=parms["doDescm"], flagVer=-1, logfile=logFile, zapin=True, check=check, debug=debug)
doneHann = True
if parms["PolCal"] and parms["doHann"]:
mess = "Hanning delay calibration scan"
printMess(mess, logFile)
delay_uv = KATHann(delay_uv, EVLAAIPSName(project), delayClass, disk, seq + 1, err, \
doDescm=parms["doDescm"], flagVer=-1, logfile=logFile, zapin=True, check=check, debug=debug)
if doneHann:
# Halve channels after hanning.
parms["selChan"] = int(parms["selChan"] / 2)
parms["BChDrop"] = int(parms["BChDrop"] / 2)
parms["EChDrop"] = int(parms["EChDrop"] / 2)
if uv == None and not check:
raise RuntimeError("Cannot Hann data ")
# Clear any old calibration/editing
if parms["doClearTab"] or kwargs.get('reuse'):
mess = "Clear previous calibration"
printMess(mess, logFile)
EVLAClearCal(uv,
err,
doGain=parms["doClearGain"],
doFlag=parms["doClearFlag"],
doBP=parms["doClearBP"],
check=check)
OErr.printErrMsg(err, "Error resetting calibration")
# Copy FG 1 to FG 2
if parms["doCopyFG"]:
mess = "Copy FG 1 to FG 2"
printMess(mess, logFile)
retCode = KATCopyFG(uv, err, logfile=logFile, check=check, debug=debug)
if retCode != 0:
raise RuntimeError("Error Copying FG table")
# Flag antennas shadowed by others?
if parms["doShad"]:
retCode = EVLAShadow (uv, err, shadBl=parms["shadBl"], \
logfile=logFile, check=check, debug=debug)
if retCode != 0:
raise RuntimeError("Error Shadow flagging data")
# Median window time editing, for RFI impulsive in time
if parms["doMednTD1"]:
mess = "Median window time editing, for RFI impulsive in time:"
printMess(mess, logFile)
retCode = EVLAMedianFlag (uv, clist, err, noScrat=noScrat, nThreads=nThreads, \
avgTime=parms["mednAvgTime"], avgFreq=parms["mednAvgFreq"], chAvg= parms["mednChAvg"], \
timeWind=parms["mednTimeWind"],flagVer=2, flagTab=2,flagSig=parms["mednSigma"], \
logfile=logFile, check=check, debug=False)
if retCode != 0:
raise RuntimeError("Error in MednFlag")
# Median window frequency editing, for RFI impulsive in frequency
if parms["doFD1"]:
mess = "Median window frequency editing, for RFI impulsive in frequency:"
printMess(mess, logFile)
retCode = EVLAAutoFlag (uv, clist, err, flagVer=2, flagTab=2, doCalib=-1, doBand=-1, \
timeAvg=parms["FD1TimeAvg"], \
doFD=True, FDmaxAmp=1.0e20, FDmaxV=1.0e20, FDwidMW=parms["FD1widMW"], \
FDmaxRMS=[1.0e20,0.1], FDmaxRes=parms["FD1maxRes"], \
FDmaxResBL= parms["FD1maxRes"], FDbaseSel=parms["FD1baseSel"],\
nThreads=nThreads, logfile=logFile, check=check, debug=debug)
if retCode != 0:
raise RuntimeError("Error in AutoFlag")
# Parallactic angle correction?
if parms["doPACor"]:
retCode = EVLAPACor(uv, err, \
logfile=logFile, check=check, debug=debug)
if retCode != 0:
raise RuntimeError("Error in Parallactic angle correction")
# Need to find a reference antenna? See if we have saved it?
if (parms["refAnt"] <= 0):
refAnt = FetchObject(fileRoot + ".refAnt.pickle")
if refAnt:
parms["refAnt"] = refAnt
# Use bandpass calibrator and center half of each spectrum
if parms["refAnt"] <= 0:
mess = "Find best reference antenna: run Calib on BP Cal(s) "
printMess(mess, logFile)
parms["refAnt"] = EVLAGetRefAnt(uv, parms["BPCals"], err, flagVer=0, \
solInt=parms["bpsolint1"], nThreads=nThreads, \
logfile=logFile, check=check, debug=debug)
if err.isErr:
raise RuntimeError("Error finding reference antenna")
if parms["refAnts"][0] <= 0:
parms["refAnts"][0] = parms["refAnt"]
mess = "Picked reference antenna " + str(parms["refAnt"])
printMess(mess, logFile)
# Save it
ParmsPicklefile = fileRoot + ".Parms.pickle" # Where results saved
SaveObject(parms, ParmsPicklefile, True)
refAntPicklefile = fileRoot + ".refAnt.pickle" # Where results saved
SaveObject(parms["refAnt"], refAntPicklefile, True)
# Plot Raw, edited data?
if parms["doRawSpecPlot"] and parms["plotSource"]:
mess = "Raw Spectral plot for: " + ' '.join(parms["BPCal"])
printMess(mess, logFile)
plotFile = fileRoot + "_RawSpec.ps"
retCode = EVLASpectrum(uv, parms["BPCal"], parms["plotTime"], maxgap, plotFile, parms["refAnt"], err, \
Stokes=["RR","LL"], doband=-1, \
check=check, debug=debug, logfile=logFile )
if retCode != 0:
raise RuntimeError("Error in Plotting spectrum")
EVLAAddOutFile(plotFile, 'project', 'Pipeline log file')
if parms["PolCal"]:
mess = "XYphase bandpass calibration"
printMess(mess, logFile)
retCode = KATXPhase(delay_uv,
uv,
err,
logfile=logFile,
check=check,
debug=debug,
doCalib=-1,
flagVer=0,
doBand=-1,
refAnt=parms['refAnt'])
doBand = 1
BPVer += 1
if retCode != 0:
raise RuntimeError("Error in Xphase calibration")
# delay calibration
if parms["doDelayCal"] and parms["DCals"] and not check:
plotFile = fileRoot + "_DelayCal.ps"
retCode = EVLADelayCal(uv, parms["DCals"], err, \
BChan=parms["delayBChan"], EChan=parms["delayEChan"], \
doCalib=-1, flagVer=0, doBand=doBand, BPVer=BPVer, \
solInt=parms["delaySolInt"], smoTime=parms["delaySmoo"], \
refAnts=[parms["refAnt"]], doTwo=parms["doTwo"],
doZeroPhs=parms["delayZeroPhs"], \
doAvgIF=parms["delayAvgIF"], doAvgPol=parms["delayAvgPol"], \
doPlot=parms["doSNPlot"], plotFile=plotFile, \
nThreads=nThreads, noScrat=noScrat, \
logfile=logFile, check=check, debug=debug)
if retCode != 0:
raise RuntimeError("Error in delay calibration")
# Plot corrected data?
if parms["doSpecPlot"] and parms["plotSource"]:
plotFile = fileRoot + "_DelaySpec.ps"
retCode = EVLASpectrum(uv, parms["BPCal"], parms["plotTime"], maxgap, \
plotFile, parms["refAnt"], err, \
Stokes=["RR","LL"], doband=doBand, \
check=check, debug=debug, logfile=logFile )
if retCode != 0:
raise RuntimeError("Error in Plotting spectrum")
# Bandpass calibration
if parms["doBPCal"] and parms["BPCals"]:
retCode = KATBPCal(uv, parms["BPCals"], err, doBand=doBand, BPVer=BPVer, newBPVer=0,
noScrat=noScrat, solInt1=parms["bpsolint1"], \
solInt2=parms["bpsolint2"], solMode=parms["bpsolMode"], \
BChan1=parms["bpBChan1"], EChan1=parms["bpEChan1"], \
BChan2=parms["bpBChan2"], EChan2=parms["bpEChan2"], ChWid2=parms["bpChWid2"], \
doCenter1=parms["bpDoCenter1"], refAnt=parms["refAnt"], \
UVRange=parms["bpUVRange"], doCalib=2, gainUse=0, flagVer=0, doPlot=False, \
nThreads=nThreads, logfile=logFile, check=check, debug=debug)
if retCode != 0:
raise RuntimeError("Error in Bandpass calibration")
# Plot corrected data?
if parms["doSpecPlot"] and parms["plotSource"]:
plotFile = fileRoot + "_BPSpec.ps"
retCode = EVLASpectrum(uv, parms["BPCal"], parms["plotTime"], maxgap, plotFile, \
parms["refAnt"], err, Stokes=["RR","LL"], doband=1, \
check=check, debug=debug, logfile=logFile )
if retCode != 0:
raise RuntimeError("Error in Plotting spectrum")
# Amp & phase Calibrate
if parms["doAmpPhaseCal"]:
plotFile = fileRoot + "_APCal.ps"
retCode = KATCalAP (uv, [], parms["ACals"], err, PCals=parms["PCals"],
doCalib=2, doBand=1, BPVer=0, flagVer=0, \
BChan=parms["ampBChan"], EChan=parms["ampEChan"], \
solInt=parms["solInt"], solSmo=parms["solSmo"], ampScalar=parms["ampScalar"], \
doAmpEdit=parms["doAmpEdit"], ampSigma=parms["ampSigma"], \
ampEditFG=parms["ampEditFG"], avgPol=parms["PolCal"], \
doPlot=parms["doSNPlot"], plotFile=plotFile, refAnt=parms["refAnt"], \
nThreads=nThreads, noScrat=noScrat, logfile=logFile, check=check, debug=debug)
if retCode != 0:
raise RuntimeError("Error calibrating")
# More editing
if parms["doAutoFlag"]:
mess = "Post calibration editing:"
printMess(mess, logFile)
# if going to redo then only calibrators
if parms["doRecal"]:
# Only calibrators
clist = []
for DCal in parms["DCals"]:
if DCal["Source"] not in clist:
clist.append(DCal["Source"])
for PCal in parms["PCals"]:
if PCal["Source"] not in clist:
clist.append(PCal["Source"])
for ACal in parms["ACals"]:
if ACal["Source"] not in clist:
clist.append(ACal["Source"])
else:
clist = []
retCode = EVLAAutoFlag (uv, clist, err, flagVer=0, flagTab =2, \
doCalib=2, gainUse=0, doBand=1, BPVer=BPVer, \
IClip=parms["IClip"], minAmp=parms["minAmp"], timeAvg=parms["timeAvg"], \
doFD=parms["doFirstAFFD"], FDmaxAmp=parms["FDmaxAmp"], FDmaxV=parms["FDmaxV"], \
FDwidMW=parms["FDwidMW"], FDmaxRMS=parms["FDmaxRMS"], \
FDmaxRes=parms["FDmaxRes"], FDmaxResBL=parms["FDmaxResBL"], \
FDbaseSel=parms["FDbaseSel"], \
nThreads=nThreads, logfile=logFile, check=check, debug=debug)
if retCode != 0:
raise RuntimeError("Error in AutoFlag")
# Redo the calibration using new flagging?
if parms["doBPCal2"] == None:
parms["doBPCal2"] = parms["doBPCal"]
if parms["doDelayCal2"] == None:
parms["doDelayCal2"] = parms["doDelayCal2"]
if parms["doAmpPhaseCal2"] == None:
parms["doAmpPhaseCal2"] = parms["doAmpPhaseCal"]
if parms["doAutoFlag2"] == None:
parms["doAutoFlagCal2"] = parms["doAutoFlag"]
if parms["doRecal"]:
mess = "Redo calibration:"
printMess(mess, logFile)
EVLAClearCal(uv,
err,
doGain=True,
doFlag=False,
doBP=True,
check=check,
logfile=logFile)
OErr.printErrMsg(err, "Error resetting calibration")
BPVer = 0
# Parallactic angle correction?
if parms["doPACor"]:
retCode = EVLAPACor(uv, err, \
logfile=logFile, check=check, debug=debug)
if retCode != 0:
raise RuntimeError("Error in Parallactic angle correction")
# Run MKXPhase on delaycal data and attach BP table to UV data
if parms["PolCal"]:
mess = "XYphase bandpass calibration"
printMess(mess, logFile)
retCode = KATXPhase(delay_uv,
uv,
err,
logfile=logFile,
check=check,
debug=debug,
doCalib=-1,
flagVer=0,
doBand=-1,
refAnt=parms['refAnt'])
BPVer += 1
if retCode != 0:
raise RuntimeError("Error in Xphase calibration")
# Delay recalibration
if parms["doDelayCal2"] and parms["DCals"] and not check:
plotFile = fileRoot + "_DelayCal2.ps"
retCode = EVLADelayCal(uv, parms["DCals"], err, \
BChan=parms["delayBChan"], EChan=parms["delayEChan"], \
doCalib=-1, flagVer=0, doBand=doBand, BPVer=BPVer, \
solInt=parms["delaySolInt"], smoTime=parms["delaySmoo"], \
refAnts=[parms["refAnt"]], doTwo=parms["doTwo"], \
doZeroPhs=parms["delayZeroPhs"], \
doAvgIF=parms["delayAvgIF"], doAvgPol=parms["delayAvgPol"], \
doPlot=parms["doSNPlot"], plotFile=plotFile, \
nThreads=nThreads, noScrat=noScrat, \
logfile=logFile, check=check, debug=debug)
if retCode != 0:
raise RuntimeError("Error in delay calibration")
# Plot corrected data?
if parms["doSpecPlot"] and parms["plotSource"]:
plotFile = fileRoot + "_DelaySpec2.ps"
retCode = EVLASpectrum(uv, parms["BPCal"], parms["plotTime"], maxgap, plotFile, parms["refAnt"], err, \
Stokes=["RR","LL"], doband=doband, \
check=check, debug=debug, logfile=logFile )
if retCode != 0:
raise RuntimeError("Error in Plotting spectrum")
# Bandpass calibration
if parms["doBPCal2"] and parms["BPCals"]:
retCode = KATBPCal(uv, parms["BPCals"], err, doBand=doBand, BPVer=BPVer, newBPVer=0, \
noScrat=noScrat, solInt1=parms["bpsolint1"], \
solInt2=parms["bpsolint2"], solMode=parms["bpsolMode"], \
BChan1=parms["bpBChan1"], EChan1=parms["bpEChan1"], \
BChan2=parms["bpBChan2"], EChan2=parms["bpEChan2"], ChWid2=parms["bpChWid2"], \
doCenter1=parms["bpDoCenter1"], refAnt=parms["refAnt"], \
UVRange=parms["bpUVRange"], doCalib=2, gainUse=0, flagVer=0, doPlot=False, \
nThreads=nThreads, logfile=logFile, check=check, debug=debug)
if retCode != 0:
raise RuntimeError("Error in Bandpass calibration")
# Plot corrected data?
if parms["doSpecPlot"] and parms["plotSource"]:
plotFile = fileRoot + "_BPSpec2.ps"
retCode = EVLASpectrum(uv, parms["BPCal"], parms["plotTime"], maxgap, plotFile, parms["refAnt"], err, \
Stokes=["RR","LL"], doband=1, \
check=check, debug=debug, logfile=logFile )
if retCode != 0:
raise RuntimeError("Error in Plotting spectrum")
# Amp & phase Recalibrate
if parms["doAmpPhaseCal2"]:
plotFile = fileRoot + "_APCal2.ps"
retCode = KATCalAP (uv, [], parms["ACals"], err, PCals=parms["PCals"], \
doCalib=2, doBand=1, BPVer=0, flagVer=0, \
BChan=parms["ampBChan"], EChan=parms["ampEChan"], \
solInt=parms["solInt"], solSmo=parms["solSmo"], ampScalar=parms["ampScalar"], \
doAmpEdit=True, ampSigma=parms["ampSigma"], \
ampEditFG=parms["ampEditFG"], avgPol=parms["PolCal"], \
doPlot=parms["doSNPlot"], plotFile=plotFile, refAnt=parms["refAnt"], \
noScrat=noScrat, nThreads=nThreads, logfile=logFile, check=check, debug=debug)
if retCode != 0:
raise RuntimeError("Error calibrating")
# More editing
if parms["doAutoFlag2"]:
mess = "Post recalibration editing:"
printMess(mess, logFile)
retCode = EVLAAutoFlag (uv, [], err, flagVer=0, flagTab=2, \
doCalib=2, gainUse=0, doBand=1, BPVer=0, \
IClip=parms["IClip"], minAmp=parms["minAmp"], timeAvg=parms["timeAvg"], \
doFD=parms["doSecAFFD"], FDmaxAmp=parms["FDmaxAmp"], FDmaxV=parms["FDmaxV"], \
FDwidMW=parms["FDwidMW"], FDmaxRMS=parms["FDmaxRMS"], \
FDmaxRes=parms["FDmaxRes"], FDmaxResBL= parms["FDmaxResBL"], \
FDbaseSel=parms["FDbaseSel"], \
nThreads=nThreads, logfile=logFile, check=check, debug=debug)
if retCode != 0:
raise RuntimeError("Error in AutoFlag")
# end recal
# Calibrate and average data
# Overwrite avgStokes from command line
if kwargs.get('halfstokes'):
parms["avgStokes"] = 'HALF'
if parms["doCalAvg"] == 'Splat':
retCode = KATCalAvg (uv, avgClass, parms["seq"], parms["CalAvgTime"], err, \
flagVer=2, doCalib=2, gainUse=0, doBand=1, BPVer=0, doPol=False, \
avgFreq=parms["avgFreq"], chAvg=parms["chAvg"], Stokes=parms["avgStokes"], \
BChan=1, EChan=parms["selChan"] - 1, doAuto=parms["doAuto"], \
BIF=parms["CABIF"], EIF=parms["CAEIF"], Compress=parms["Compress"], \
nThreads=nThreads, logfile=logFile, check=check, debug=debug)
if retCode != 0:
raise RuntimeError("Error in CalAvg")
elif parms["doCalAvg"] == 'BL':
retCode = KATBLCalAvg (uv, avgClass, parms["seq"], err, \
flagVer=2, doCalib=2, gainUse=0, doBand=1, BPVer=0, doPol=False, \
avgFreq=parms["avgFreq"], chAvg=parms["chAvg"], FOV=parms['FOV'], \
maxInt=min(parms["solPInt"],parms["solAInt"]), Stokes=parms["avgStokes"], \
BChan=1, EChan=parms["selChan"] - 1, timeAvg=parms["CalAvgTime"], \
BIF=parms["CABIF"], EIF=parms["CAEIF"], Compress=parms["Compress"], \
logfile=logFile, check=check, debug=debug)
if retCode != 0:
raise RuntimeError("Error in BLCalAvg")
if parms["doSaveTab"]:
filename = project + ".CalTab.uvtab"
_ = EVLAUVFITSTab(uv, filename, 0, err, logfile=logFile)
#Zap unaveraged data if requested
if kwargs.get('zapraw'):
uv.Zap(err)
# Get calibrated/averaged data
if not check:
uv = UV.newPAUV("AIPS UV DATA", EVLAAIPSName(project), avgClass[0:6], \
disk, parms["seq"], True, err)
if err.isErr:
OErr.printErrMsg(err, "Error creating cal/avg AIPS data")
plotFile = fileRoot + "_Spec.ps"
retCode = EVLASpectrum(uv, parms["BPCal"], parms["plotTime"], maxgap, \
plotFile, parms["refAnt"], err, \
Stokes=["I"], doband=-1, docalib=-1, \
check=check, debug=debug, logfile=logFile )
if retCode != 0:
raise RuntimeError("Error in Plotting spectrum")
# KATUVFITS(uv, 'preimage.uvfits', 0, err, exclude=["AIPS HI", "AIPS SL", "AIPS PL"],
# include=["AIPS AN", "AIPS FQ"], compress=parms["Compress"], logfile=logFile)
KATUVFITab(uv, project + '.uvtab', 0, err)
#Gzip the data?
if kwargs.get('gzip'):
os.system('pigz -p %d %s' % (nThreads, project + '.uvtab'))
os.system('rm -f %s' % (project + '.uvtab'))
def AIPSSetup(scratchdir=None):
cwd = './'
home = os.path.expanduser("~")
# Is Obit installed and set up correctly? If not make data cwd/FITS.
try:
OBIT_EXEC = os.environ['OBIT']
OBIT_DATA = OBIT_EXEC + '/share/data'
except:
OBIT_EXEC = cwd
OBIT_DATA = OBIT_EXEC + '/FITS'
#AIPS user number (arbitrary)
user = 10
# Check if AIPS is defined and use it as the default if it is.
try:
aips_dir = os.environ['AIPS_ROOT']
aips_version = os.environ['AIPS_VERSION'][-7:]
except:
aips_dir = cwd
aips_version = '31DEC18' # Should sort out where to change this if necessary!!
configdefaults = {
'aips_dir': aips_dir,
'aips_version': aips_version,
'scratch_area': cwd,
'metadata_dir': OBIT_DATA
}
config = ConfigParser.ConfigParser(configdefaults)
config.add_section('KATPIPE')
# Get the config file in the users home directory if it exists and overwrite the defaults
config.read(home + '/.katimrc')
if scratchdir:
config.set('KATPIPE', 'scratch_area', scratchdir)
############################# Initialize AIPS ##########################################
# Sort out the AIPS environment variables for the defined configuration.
AIPSLite.get_aips(basedir=config.get('KATPIPE', 'aips_dir'),
version=config.get('KATPIPE', 'aips_version'))
# Make the aips scratch disk
DA00 = config.get('KATPIPE', 'scratch_area') + '/da00'
AIPS_DISK = config.get('KATPIPE', 'scratch_area') + '/aipsdisk'
#Overwrite any previous AIPS disks.
if os.path.exists(DA00): shutil.rmtree(DA00)
if os.path.exists(AIPS_DISK): shutil.rmtree(AIPS_DISK)
# Create the aips disk and the AIPS environment for the disks.
AIPSLite.make_disk(disk_path=AIPS_DISK)
AIPSLite.filaip(force=True, data_dir=AIPS_DISK)
AIPSLite.make_da00(da00_path=DA00)
# Get the set up AIPS environment.
AIPS_ROOT = os.environ['AIPS_ROOT']
AIPS_VERSION = os.environ['AIPS_VERSION']
adirs = [(None, AIPS_DISK)]
fdirs = [(None, config.get('KATPIPE', 'metadata_dir'))]
############################# Initialize OBIT ##########################################
err = OErr.OErr()
ffdirs = []
for f in fdirs:
ffdirs.append(f[1])
aadirs = []
for a in adirs:
aadirs.append(a[1])
ObitSys = OSystem.OSystem ("Pipeline", 1, user, len(aadirs), aadirs, \
len(ffdirs), ffdirs, True, False, err)
OErr.printErrMsg(err, "Error with Obit startup")
# setup environment
ObitTalkUtil.SetEnviron(AIPS_ROOT=AIPS_ROOT, AIPS_VERSION=AIPS_VERSION, \
OBIT_EXEC=OBIT_EXEC, DA00=DA00, ARCH=AIPSLite.arch(), \
aipsdirs=adirs, fitsdirs=fdirs)
# List directories
ObitTalkUtil.ListAIPSDirs()
ObitTalkUtil.ListFITSDirs()
# Disks to avoid
noScrat = [0] # AIPS disks to avoid
return ObitSys
def __init__(self, data):
self._err = OErr.OErr()
self._table = History.History('AIPS HI', data.List, self._err)
self._table.Open(3, self._err)
if self._err.isErr:
raise RuntimeError
def MKContPipeline(files, outputdir, **kwargs):
"""MeerKAT Continuum pipeline.
Parameters
----------
files : list
h5 filenames (note: support for multiple h5 files
i.e. ConcatenatedDataSet is not currently supported)
outputdir : string
Directory location to write output data,
scratchdir : string, optional
The directory location of the aips disk
parmFile : string, optional
Overwrite the default imaging parameters using this parameter file.
"""
#if len(files) > 1:
# raise TooManyKatfilesException('Processing multiple katfiles are not currently supported')
# Onle be concatenated if we have to be
if len(files) == 1:
h5file = files[0]
else:
h5file = files
# Die gracefully if we cannot write to the output area...
if not os.path.exists(outputdir):
print('Specified output directory: ' + outputdir + 'does not exist.')
exit(-1)
# Obit error logging
err = OErr.OErr()
#################### Initialize filenames #######################################################
fileRoot = os.path.join(outputdir,
os.path.basename(os.path.splitext(
files[0])[0])) # root of file name
logFile = fileRoot + ".log" # Processing log file
avgClass = ("UVAv")[0:6] # Averaged data AIPS class
manifestfile = outputdir + '/manifest.pickle'
############################# Initialize OBIT and AIPS ##########################################
noScrat = []
# Logging directly to logFile
OErr.PInit(err, 2, logFile)
EVLAAddOutFile(os.path.basename(logFile), 'project', 'Pipeline log file')
if kwargs.get('reuse'):
ObitSys = AIPSSetup.AIPSSetup(err,
configfile=kwargs.get('configFile'),
scratchdir=kwargs.get('scratchdir'),
aipsdisk=kwargs.get('aipsdisk'),
overwrite=False)
else:
ObitSys = AIPSSetup.AIPSSetup(err,
configfile=kwargs.get('configFile'),
scratchdir=kwargs.get('scratchdir'),
aipsdisk=kwargs.get('aipsdisk'))
# Get the set up AIPS environment.
AIPS_ROOT = os.environ['AIPS_ROOT']
AIPS_VERSION = os.environ['AIPS_VERSION']
nThreads = 72
user = OSystem.PGetAIPSuser()
AIPS.userno = user
disk = 1
fitsdisk = 0
nam = os.path.basename(os.path.splitext(files[0])[0])[0:10]
cls = "Raw"
seq = 1
############################# Initialise Parameters ##########################################
####### Initialize parameters dictionary #####
parms = KATInitContParms()
####### User defined parameters ######
if kwargs.get('parmFile'):
print("parmFile", kwargs.get('parmFile'))
exec(open(kwargs.get('parmFile')).read())
EVLAAddOutFile(os.path.basename(kwargs.get('parmFile')), 'project',
'Pipeline input parameters')
############### Initialize katfile object, uvfits object and condition data #########################
OK = False
# Open the h5 file as a katfile object
try:
#open katfile and perform selection according to kwargs
katdata = katfile.open(h5file)
OK = True
except Exception as exception:
print(exception)
if not OK:
OErr.PSet(err)
OErr.PLog(err, OErr.Fatal,
"Unable to read KAT HDF5 data in " + str(h5file))
raise KATUnimageableError("Unable to read KAT HDF5 data in " +
str(h5file))
#Are we MeerKAT or KAT-7
telescope = katdata.ants[0].name[0]
if telescope == 'm':
sefd = 500.
else:
sefd = 1200.
#Get calibrator models
fluxcals = katpoint.Catalogue(
open(FITSDir.FITSdisks[0] + "/" + parms["fluxModel"]))
#Condition data (get bpcals, update names for aips conventions etc)
KATh5Condition(katdata, fluxcals, err)
###################### Data selection and static edits ############################################
# Select data based on static imageable parameters
MKATh5Select(katdata, parms, err, **kwargs)
# General AIPS data parameters at script level
dataClass = ("UVDa")[0:6] # AIPS class of raw uv data
band = katdata.spectral_windows[0].product #Correlator product
project = os.path.basename(os.path.splitext(files[0])[0])[
0:10] # Project name (12 char or less, used as AIPS Name)
outIClass = parms["outIClass"] # image AIPS class
debug = parms["debug"]
check = parms["check"]
####################### Import data into AIPS #####################################################
# Reuse or nay?
if kwargs.get('reuse'):
uv = UV.newPAUV("AIPS UV DATA", EVLAAIPSName(project), dataClass, disk,
seq, True, err)
obsdata = KATH5toAIPS.GetKATMeta(katdata, err)
# Extract AIPS parameters of the uv data to the metadata
obsdata["Aproject"] = uv.Aname
obsdata["Aclass"] = uv.Aclass
obsdata["Aseq"] = uv.Aseq
obsdata["Adisk"] = disk
obsdata["calInt"] = katdata.dump_period
obsdata["fitsdisk"] = fitsdisk
# TODO: Check if the input data has been Hanned.
doneHann = True
else:
# Number of baselines gives batch size
nbl = len(
np.unique([(cp[0][:-1] + cp[1][:-1]).upper()
for cp in katdata.corr_products]))
# Construct a template uvfits file from master template
mastertemplate = ObitTalkUtil.FITSDir.FITSdisks[
fitsdisk] + 'MKATTemplate.uvtab.gz'
outtemplate = nam + '.uvtemp'
KATH5toAIPS.MakeTemplate(mastertemplate,
outtemplate,
len(katdata.channel_freqs),
nvispio=nbl)
uv = OTObit.uvlod(outtemplate, 0, EVLAAIPSName(project), cls, disk,
seq, err)
obsdata = KATH5toAIPS.KAT2AIPS(katdata,
uv,
disk,
fitsdisk,
err,
calInt=katdata.dump_period,
**kwargs)
MakeIFs.UVMakeIF(uv, 8, err)
os.remove(outtemplate)
# Print the uv data header to screen.
uv.Header(err)
############################# Set Project Processing parameters ###################################
# Parameters derived from obsdata and katdata
MKATGetObsParms(obsdata, katdata, parms, logFile)
###### Initialise target parameters #####
KATInitTargParms(katdata, parms, err)
# Load the outputs pickle jar
EVLAFetchOutFiles()
OSystem.PAllowThreads(nThreads) # Allow threads in Obit/oython
retCode = 0
maxgap = max(parms["CalAvgTime"], 20 * katdata.dump_period) / 60.
################### Start processing ###############################################################
mess = "Start project "+parms["project"]+" AIPS user no. "+str(AIPS.userno)+\
", KAT7 configuration "+parms["KAT7Cfg"]
printMess(mess, logFile)
if debug:
pydoc.ttypager = pydoc.plainpager # don't page task input displays
mess = "Using Debug mode "
printMess(mess, logFile)
if check:
mess = "Only checking script"
printMess(mess, logFile)
# Log parameters
printMess("Parameter settings", logFile)
for p in parms:
mess = " " + p + ": " + str(parms[p])
printMess(mess, logFile)
clist = []
for DCal in parms["DCals"]:
if DCal["Source"] not in clist:
clist.append(DCal["Source"])
for PCal in parms["PCals"]:
if PCal["Source"] not in clist:
clist.append(PCal["Source"])
for ACal in parms["ACals"]:
if ACal["Source"] not in clist:
clist.append(ACal["Source"])
if kwargs.get('targets') is not None:
targets = [
targ.name for targ in katdata.catalogue
if (targ.name not in clist) and (
targ.name in kwargs.get('targets').split(','))
]
else:
targets = [
targ.name for targ in katdata.catalogue if (targ.name not in clist)
]
refAnt = FetchObject(fileRoot + ".refAnt.pickle")
# Save parameters to pickle jar, manifest
ParmsPicklefile = fileRoot + ".Parms.pickle" # Where results saved
SaveObject(parms, ParmsPicklefile, True)
EVLAAddOutFile(os.path.basename(ParmsPicklefile), 'project',
'Processing parameters used')
loadClass = dataClass
# Hanning - No Hanning
parms["doHann"] = False
doneHann = False
if doneHann:
# Halve channels after hanning.
parms["selChan"] = int(parms["selChan"] / 2)
parms["BChDrop"] = int(parms["BChDrop"] / 2)
parms["EChDrop"] = int(parms["EChDrop"] / 2)
if uv == None and not check:
raise RuntimeError("Cannot Hann data ")
# Clear any old calibration/editing
if parms["doClearTab"] or kwargs.get('reuse'):
mess = "Clear previous calibration"
printMess(mess, logFile)
EVLAClearCal(uv,
err,
doGain=parms["doClearGain"],
doFlag=parms["doClearFlag"],
doBP=parms["doClearBP"],
check=check)
OErr.printErrMsg(err, "Error resetting calibration")
# Quack to remove data from start and end of each scan
if parms["doQuack"]:
retCode = EVLAQuack (uv, err, begDrop=parms["quackBegDrop"], endDrop=parms["quackEndDrop"], \
Reason=parms["quackReason"], \
logfile=logFile, check=check, debug=debug)
if retCode != 0:
raise RuntimeError("Error Quacking data")
# Flag antennas shadowed by others?
if parms["doShad"]:
retCode = EVLAShadow (uv, err, shadBl=parms["shadBl"], \
logfile=logFile, check=check, debug=debug)
if retCode != 0:
raise RuntimeError("Error Shadow flagging data")
# Median window time editing, for RFI impulsive in time
if parms["doMednTD1"]:
mess = "Median window time editing, for RFI impulsive in time:"
printMess(mess, logFile)
retCode = EVLAMedianFlag (uv, clist, err, noScrat=noScrat, nThreads=nThreads, \
avgTime=parms["mednAvgTime"], avgFreq=parms["mednAvgFreq"], chAvg= parms["mednChAvg"], \
timeWind=parms["mednTimeWind"],flagVer=2, flagTab=2,flagSig=parms["mednSigma"], \
logfile=logFile, check=check, debug=False)
if retCode != 0:
raise RuntimeError("Error in MednFlag")
# Median window frequency editing, for RFI impulsive in frequency
if parms["doFD1"]:
mess = "Median window frequency editing, for RFI impulsive in frequency:"
printMess(mess, logFile)
retCode = EVLAAutoFlag (uv, clist, err, flagVer=2, flagTab=2, doCalib=-1, doBand=-1, \
timeAvg=parms["FD1TimeAvg"], \
doFD=True, FDmaxAmp=1.0e20, FDmaxV=1.0e20, FDwidMW=parms["FD1widMW"], \
FDmaxRMS=[1.0e20,0.1], FDmaxRes=parms["FD1maxRes"], \
FDmaxResBL= parms["FD1maxRes"], FDbaseSel=parms["FD1baseSel"],\
nThreads=nThreads, logfile=logFile, check=check, debug=debug)
if retCode != 0:
raise RuntimeError("Error in AutoFlag")
# Parallactic angle correction?
if parms["doPACor"]:
retCode = EVLAPACor(uv, err, \
logfile=logFile, check=check, debug=debug)
if retCode != 0:
raise RuntimeError("Error in Parallactic angle correction")
# Need to find a reference antenna? See if we have saved it?
if (parms["refAnt"] <= 0):
refAnt = FetchObject(fileRoot + ".refAnt.pickle")
if refAnt:
parms["refAnt"] = refAnt
# Use bandpass calibrator and center half of each spectrum
if parms["refAnt"] <= 0:
mess = "Find best reference antenna: run Calib on BP Cal(s) "
printMess(mess, logFile)
parms["refAnt"] = EVLAGetRefAnt(uv, parms["BPCals"], err, flagVer=0, \
solInt=parms["bpsolint1"], nThreads=nThreads, \
logfile=logFile, check=check, debug=debug)
if err.isErr:
raise RuntimeError("Error finding reference antenna")
if parms["refAnts"][0] <= 0:
parms["refAnts"][0] = parms["refAnt"]
mess = "Picked reference antenna " + str(parms["refAnt"])
printMess(mess, logFile)
# Save it
ParmsPicklefile = fileRoot + ".Parms.pickle" # Where results saved
SaveObject(parms, ParmsPicklefile, True)
refAntPicklefile = fileRoot + ".refAnt.pickle" # Where results saved
SaveObject(parms["refAnt"], refAntPicklefile, True)
# Plot Raw, edited data?
parms["doRawSpecPlot"] = False
parms["doSpecPlot"] = False
if parms["doRawSpecPlot"] and parms["plotSource"]:
mess = "Raw Spectral plot for: " + ' '.join(parms["BPCal"])
printMess(mess, logFile)
plotFile = fileRoot + "_RawSpec.ps"
retCode = EVLASpectrum(uv, parms["BPCal"], parms["plotTime"], maxgap, plotFile, parms["refAnt"], err, \
Stokes=["RR","LL"], doband=-1, \
check=check, debug=debug, logfile=logFile )
if retCode != 0:
raise RuntimeError("Error in Plotting spectrum")
EVLAAddOutFile(plotFile, 'project', 'Pipeline log file')
# delay calibration
if parms["doDelayCal"] and parms["DCals"] and not check:
plotFile = fileRoot + "_DelayCal.ps"
retCode = EVLADelayCal(uv, parms["DCals"], err, \
BChan=parms["delayBChan"], EChan=parms["delayEChan"], \
doCalib=2, flagVer=0, doBand=-1, \
solInt=parms["delaySolInt"], smoTime=parms["delaySmoo"], \
refAnts=[parms["refAnt"]], doTwo=parms["doTwo"],
doZeroPhs=parms["delayZeroPhs"], \
doPlot=parms["doSNPlot"], plotFile=plotFile, \
nThreads=nThreads, noScrat=noScrat, \
logfile=logFile, check=check, debug=debug)
if retCode != 0:
raise RuntimeError("Error in delay calibration")
# Plot corrected data?
if parms["doSpecPlot"] and parms["plotSource"]:
plotFile = fileRoot + "_DelaySpec.ps"
retCode = EVLASpectrum(uv, parms["BPCal"], parms["plotTime"], maxgap, \
plotFile, parms["refAnt"], err, \
Stokes=["RR","LL"], doband=-1, \
check=check, debug=debug, logfile=logFile )
if retCode != 0:
raise RuntimeError("Error in Plotting spectrum")
print(parms["bpBChan1"], parms["bpEChan1"], parms["bpBChan2"],
parms["bpEChan2"], parms["bpChWid2"])
# Bandpass calibration
if parms["doBPCal"] and parms["BPCals"]:
retCode = KATBPCal(uv, parms["BPCals"], err, noScrat=noScrat, solInt1=parms["bpsolint1"], \
solInt2=parms["bpsolint2"], solMode=parms["bpsolMode"], \
BChan1=parms["bpBChan1"], EChan1=parms["bpEChan1"], \
BChan2=parms["bpBChan2"], EChan2=parms["bpEChan2"], ChWid2=parms["bpChWid2"], \
doCenter1=parms["bpDoCenter1"], refAnt=parms["refAnt"], \
UVRange=parms["bpUVRange"], doCalib=2, gainUse=0, flagVer=0, doPlot=False, \
nThreads=nThreads, logfile=logFile, check=check, debug=debug)
if retCode != 0:
raise RuntimeError("Error in Bandpass calibration")
# Plot corrected data?
if parms["doSpecPlot"] and parms["plotSource"]:
plotFile = fileRoot + "_BPSpec.ps"
retCode = EVLASpectrum(uv, parms["BPCal"], parms["plotTime"], maxgap, plotFile, \
parms["refAnt"], err, Stokes=["RR","LL"], doband=2, \
check=check, debug=debug, logfile=logFile )
if retCode != 0:
raise RuntimeError("Error in Plotting spectrum")
# Amp & phase Calibrate
if parms["doAmpPhaseCal"]:
plotFile = fileRoot + "_APCal.ps"
retCode = KATCalAP (uv, [], parms["ACals"], err, PCals=parms["PCals"],
doCalib=2, doBand=2, BPVer=1, flagVer=0, \
BChan=parms["ampBChan"], EChan=parms["ampEChan"], \
solInt=parms["solInt"], solSmo=parms["solSmo"], ampScalar=parms["ampScalar"], \
doAmpEdit=parms["doAmpEdit"], ampSigma=parms["ampSigma"], \
ampEditFG=parms["ampEditFG"], \
doPlot=parms["doSNPlot"], plotFile=plotFile, refAnt=parms["refAnt"], \
nThreads=nThreads, noScrat=noScrat, logfile=logFile, check=check, debug=debug)
#print parms["ACals"],parms["PCals"]
if retCode != 0:
raise RuntimeError("Error calibrating")
# More editing
if parms["doAutoFlag"]:
mess = "Post calibration editing:"
printMess(mess, logFile)
# if going to redo then only calibrators
if parms["doRecal"]:
# Only calibrators
clist = []
for DCal in parms["DCals"]:
if DCal["Source"] not in clist:
clist.append(DCal["Source"])
for PCal in parms["PCals"]:
if PCal["Source"] not in clist:
clist.append(PCal["Source"])
for ACal in parms["ACals"]:
if ACal["Source"] not in clist:
clist.append(ACal["Source"])
else:
clist = []
retCode = EVLAAutoFlag (uv, clist, err, flagVer=0, flagTab =2, \
doCalib=2, gainUse=0, doBand=2, BPVer=1, \
IClip=parms["IClip"], minAmp=parms["minAmp"], timeAvg=parms["timeAvg"], \
doFD=parms["doFirstAFFD"], FDmaxAmp=parms["FDmaxAmp"], FDmaxV=parms["FDmaxV"], \
FDwidMW=parms["FDwidMW"], FDmaxRMS=parms["FDmaxRMS"], \
FDmaxRes=parms["FDmaxRes"], FDmaxResBL=parms["FDmaxResBL"], \
FDbaseSel=parms["FDbaseSel"], \
nThreads=nThreads, logfile=logFile, check=check, debug=debug)
if retCode != 0:
raise RuntimeError("Error in AutoFlag")
# Redo the calibration using new flagging?
if parms["doBPCal2"] == None:
parms["doBPCal2"] = parms["doBPCal"]
if parms["doDelayCal2"] == None:
parms["doDelayCal2"] = parms["doDelayCal2"]
if parms["doAmpPhaseCal2"] == None:
parms["doAmpPhaseCal2"] = parms["doAmpPhaseCal"]
if parms["doAutoFlag2"] == None:
parms["doAutoFlagCal2"] = parms["doAutoFlag"]
if parms["doRecal"]:
mess = "Redo calibration:"
printMess(mess, logFile)
EVLAClearCal(uv,
err,
doGain=True,
doFlag=False,
doBP=True,
check=check,
logfile=logFile)
OErr.printErrMsg(err, "Error resetting calibration")
# Parallactic angle correction?
if parms["doPACor"]:
retCode = EVLAPACor(uv, err, \
logfile=logFile, check=check, debug=debug)
if retCode != 0:
raise RuntimeError("Error in Parallactic angle correction")
# Delay recalibration
if parms["doDelayCal2"] and parms["DCals"] and not check:
plotFile = fileRoot + "_DelayCal2.ps"
retCode = EVLADelayCal(uv, parms["DCals"], err, \
BChan=parms["delayBChan"], EChan=parms["delayEChan"], \
doCalib=2, flagVer=0, doBand=-1, \
solInt=parms["delaySolInt"], smoTime=parms["delaySmoo"], \
refAnts=[parms["refAnt"]], doTwo=parms["doTwo"], \
doZeroPhs=parms["delayZeroPhs"], \
doPlot=parms["doSNPlot"], plotFile=plotFile, \
nThreads=nThreads, noScrat=noScrat, \
logfile=logFile, check=check, debug=debug)
if retCode != 0:
raise RuntimeError("Error in delay calibration")
# Plot corrected data?
if parms["doSpecPlot"] and parms["plotSource"]:
plotFile = fileRoot + "_DelaySpec2.ps"
retCode = EVLASpectrum(uv, parms["BPCal"], parms["plotTime"], maxgap, plotFile, parms["refAnt"], err, \
Stokes=["RR","LL"], doband=-1, \
check=check, debug=debug, logfile=logFile )
if retCode != 0:
raise RuntimeError("Error in Plotting spectrum")
# Bandpass calibration
if parms["doBPCal2"] and parms["BPCals"]:
retCode = KATBPCal(uv, parms["BPCals"], err, noScrat=noScrat, solInt1=parms["bpsolint1"], \
solInt2=parms["bpsolint2"], solMode=parms["bpsolMode"], \
BChan1=parms["bpBChan1"], EChan1=parms["bpEChan1"], \
BChan2=parms["bpBChan2"], EChan2=parms["bpEChan2"], ChWid2=parms["bpChWid2"], \
doCenter1=parms["bpDoCenter1"], refAnt=parms["refAnt"], \
UVRange=parms["bpUVRange"], doCalib=2, gainUse=0, flagVer=0, doPlot=False, \
nThreads=nThreads, logfile=logFile, check=check, debug=debug)
if retCode != 0:
raise RuntimeError("Error in Bandpass calibration")
# Plot corrected data?
if parms["doSpecPlot"] and parms["plotSource"]:
plotFile = fileRoot + "_BPSpec2.ps"
retCode = EVLASpectrum(uv, parms["BPCal"], parms["plotTime"], maxgap, plotFile, parms["refAnt"], err, \
Stokes=["RR","LL"], doband=2, \
check=check, debug=debug, logfile=logFile )
if retCode != 0:
raise RuntimeError("Error in Plotting spectrum")
# Amp & phase Recalibrate
if parms["doAmpPhaseCal2"]:
plotFile = fileRoot + "_APCal2.ps"
retCode = KATCalAP (uv, [], parms["ACals"], err, PCals=parms["PCals"], \
doCalib=2, doBand=2, BPVer=1, flagVer=0, \
BChan=parms["ampBChan"], EChan=parms["ampEChan"], \
solInt=parms["solInt"], solSmo=parms["solSmo"], ampScalar=parms["ampScalar"], \
doAmpEdit=True, ampSigma=parms["ampSigma"], \
ampEditFG=parms["ampEditFG"], \
doPlot=parms["doSNPlot"], plotFile=plotFile, refAnt=parms["refAnt"], \
noScrat=noScrat, nThreads=nThreads, logfile=logFile, check=check, debug=debug)
if retCode != 0:
raise RuntimeError("Error calibrating")
# More editing
parms["doAutoFlag2"] = False
if parms["doAutoFlag2"]:
mess = "Post recalibration editing:"
printMess(mess, logFile)
retCode = EVLAAutoFlag (uv, [], err, flagVer=0, flagTab=2, \
doCalib=2, gainUse=0, doBand=2, BPVer=1, \
IClip=parms["IClip"], minAmp=parms["minAmp"], timeAvg=parms["timeAvg"], \
doFD=parms["doSecAFFD"], FDmaxAmp=parms["FDmaxAmp"], FDmaxV=parms["FDmaxV"], \
FDwidMW=parms["FDwidMW"], FDmaxRMS=parms["FDmaxRMS"], \
FDmaxRes=parms["FDmaxRes"], FDmaxResBL= parms["FDmaxResBL"], \
FDbaseSel=parms["FDbaseSel"], \
nThreads=nThreads, logfile=logFile, check=check, debug=debug)
if retCode != 0:
raise RuntimeError("Error in AutoFlag")
# end recal
# Calibrate and average data
# Overwrite avgStokes from command line
parms["avgFreq"] = 0
parms["chAvg"] = 1
parms["doCalAvg"] = 'Splat'
if kwargs.get('halfstokes'):
parms["avgStokes"] = 'HALF'
if parms["doCalAvg"] == 'Splat':
retCode = KATCalAvg (uv, avgClass, parms["seq"], parms["CalAvgTime"], err, \
flagVer=2, doCalib=2, gainUse=0, doBand=2, BPVer=1, doPol=False, \
avgFreq=parms["avgFreq"], chAvg=parms["chAvg"], Stokes=parms["avgStokes"], \
BChan=1, EChan=parms["selChan"] - 1, doAuto=parms["doAuto"], \
BIF=parms["CABIF"], EIF=parms["CAEIF"], Compress=parms["Compress"], \
nThreads=nThreads, logfile=logFile, check=check, debug=debug)
if retCode != 0:
raise RuntimeError("Error in CalAvg")
elif parms["doCalAvg"] == 'BL':
retCode = KATBLCalAvg (uv, avgClass, parms["seq"], err, \
flagVer=2, doCalib=2, gainUse=0, doBand=2, BPVer=1, doPol=False, \
avgFreq=parms["avgFreq"], chAvg=parms["chAvg"], FOV=parms['FOV'], \
maxInt=min(parms["solPInt"],parms["solAInt"]), Stokes=parms["avgStokes"], \
BChan=1, EChan=parms["selChan"] - 1, timeAvg=parms["CalAvgTime"], \
BIF=parms["CABIF"], EIF=parms["CAEIF"], Compress=parms["Compress"], \
logfile=logFile, check=check, debug=debug)
if retCode != 0:
raise RuntimeError("Error in BLCalAvg")
if parms["doSaveTab"]:
filename = project + ".CalTab.uvtab"
_ = EVLAUVFITSTab(uv, filename, 0, err, logfile=logFile)
#Zap unaveraged data if requested
if kwargs.get('zapraw'):
uv.Zap(err)
# Get calibrated/averaged data
if not check:
uv = UV.newPAUV("AIPS UV DATA", EVLAAIPSName(project), avgClass[0:6], \
disk, parms["seq"], True, err)
if err.isErr:
OErr.printErrMsg(err, "Error creating cal/avg AIPS data")
plotFile = fileRoot + "_Spec.ps"
retCode = EVLASpectrum(uv, parms["BPCal"], parms["plotTime"], maxgap, \
plotFile, parms["refAnt"], err, \
Stokes=["I"], doband=-1, docalib=-1, \
check=check, debug=debug, logfile=logFile )
if retCode != 0:
raise RuntimeError("Error in Plotting spectrum")
# KATUVFITS(uv, 'preimage.uvfits', 0, err, exclude=["AIPS HI", "AIPS SL", "AIPS PL"],
# include=["AIPS AN", "AIPS FQ"], compress=parms["Compress"], logfile=logFile)
KATUVFITab(uv, project + '.uvtab', 0, err)
#Gzip the data?
if kwargs.get('gzip'):
os.system('pigz -p %d %s' % (nThreads, project + '.uvtab'))
os.system('rm -f %s' % (project + '.uvtab'))
def __init__(self):
self.err = OErr.OErr()
return
def __init__(self, desc=None):
self.err = OErr.OErr()
self.doInit = False
self.ObitSys = None
return
def __init__(self):
self.err = OErr.OErr()
#self.sys = OSystem.OSystem("ObitTalk", 1, 1, -1, [], -1, [],
# True, False, self.err)
return
def MKContPipeline(files, outputdir, **kwargs):
"""MeerKAT Continuum pipeline.
Parameters
----------
files : list
h5 filenames (note: support for multiple h5 files
i.e. ConcatenatedDataSet is not currently supported)
outputdir : string
Directory location to write output data,
scratchdir : string, optional
The directory location of the aips disk
parmFile : string, optional
Overwrite the default imaging parameters using this parameter file.
"""
#if len(files) > 1:
# raise TooManyKatfilesException('Processing multiple katfiles are not currently supported')
h5file = files
# Die gracefully if we cannot write to the output area...
if not os.path.exists(outputdir):
print('Specified output directory: '+ outputdir + 'does not exist.')
exit(-1)
# Obit error logging
err = OErr.OErr()
#################### Initialize filenames #######################################################
fileRoot = os.path.join(outputdir, os.path.basename(os.path.splitext(files[0])[0])) # root of file name
logFile = fileRoot+".log" # Processing log file
avgClass = ("UVAv")[0:6] # Averaged data AIPS class
manifestfile = outputdir + '/manifest.pickle'
############################# Initialize OBIT and AIPS ##########################################
noScrat = []
# Logging directly to logFile
OErr.PInit(err, 2, logFile)
EVLAAddOutFile(os.path.basename(logFile), 'project', 'Pipeline log file')
ObitSys = AIPSSetup.AIPSSetup(err,configfile=kwargs.get('configFile'),scratchdir=kwargs.get('scratchdir'))
# Get the set up AIPS environment.
AIPS_ROOT = os.environ['AIPS_ROOT']
AIPS_VERSION = os.environ['AIPS_VERSION']
nThreads = 24
user = OSystem.PGetAIPSuser()
AIPS.userno = user
disk = 1
fitsdisk = 0
nam = os.path.basename(os.path.splitext(files[0])[0])[0:10]
cls = "Raw"
seq = 1
############################# Initialise Parameters ##########################################
####### Initialize parameters dictionary #####
parms = KATInitContParms()
####### User defined parameters ######
if kwargs.get('parmFile'):
print("parmFile",kwargs.get('parmFile'))
exec(open(kwargs.get('parmFile')).read())
EVLAAddOutFile(os.path.basename(kwargs.get('parmFile')), 'project', 'Pipeline input parameters' )
############### Initialize katfile object, uvfits object and condition data #########################
OK = False
# Open the h5 file as a katfile object
try:
#open katfile and perform selection according to kwargs
katdata = katfile.open(h5file)
OK = True
except Exception as exception:
print(exception)
if not OK:
OErr.PSet(err)
OErr.PLog(err, OErr.Fatal, "Unable to read KAT HDF5 data in " + str(h5file))
raise KATUnimageableError("Unable to read KAT HDF5 data in " + str(h5file))
#We have a katdal object- read some flags and ad them in in available
if kwargs.get('flags') is not None:
flags=kwargs.get('flags')
fa = flags.split(',')
for fn,ff in enumerate(fa):
ex_flags_file = h5py.File(ff)
ex_flags = da.from_array(ex_flags_file['flags'], chunks=(1,342,katdata.shape[2]))
#Sum the new flags
katdata.datasets[fn].source.data.flags = ex_flags
#Are we MeerKAT or KAT-7
telescope = katdata.ants[0].name[0]
if telescope=='m':
sefd=500.
else:
sefd=1200.
#Get calibrator models
fluxcals = katpoint.Catalogue(file(FITSDir.FITSdisks[0]+"/"+parms["fluxModel"]))
#Condition data (get bpcals, update names for aips conventions etc)
KATh5Condition(katdata,fluxcals,err)
###################### Data selection and static edits ############################################
# Select data based on static imageable parameters
MKATh5Select(katdata, parms, err, **kwargs)
####################### Import data into AIPS #####################################################
# Construct a template uvfits file from master template
mastertemplate=ObitTalkUtil.FITSDir.FITSdisks[fitsdisk]+'MKATTemplate.uvtab.gz'
outtemplate=nam+'.uvtemp'
KATH5toAIPS.MakeTemplate(mastertemplate,outtemplate,len(katdata.channel_freqs))
uv=OTObit.uvlod(outtemplate,0,nam,cls,disk,seq,err)
obsdata = KATH5toAIPS.KAT2AIPS(katdata, uv, disk, fitsdisk, err, calInt=1.0, **kwargs)
MakeIFs.UVMakeIF(uv,8,err)
# Print the uv data header to screen.
uv.Header(err)
os.remove(outtemplate)
############################# Set Project Processing parameters ###################################
# Parameters derived from obsdata and katdata
MKATGetObsParms(obsdata, katdata, parms, logFile)
###### Initialise target parameters #####
KATInitTargParms(katdata,parms,err)
# General AIPS data parameters at script level
dataClass = ("UVDa")[0:6] # AIPS class of raw uv data
band = katdata.spectral_windows[0].product #Correlator product
project = parms["project"][0:12] # Project name (12 char or less, used as AIPS Name)
outIClass = parms["outIClass"] # image AIPS class
debug = parms["debug"]
check = parms["check"]
# Load the outputs pickle jar
EVLAFetchOutFiles()
OSystem.PAllowThreads(nThreads) # Allow threads in Obit/oython
retCode = 0
maxgap = max(parms["CalAvgTime"],20*katdata.dump_period)/60.
################### Start processing ###############################################################
mess = "Start project "+parms["project"]+" AIPS user no. "+str(AIPS.userno)+\
", KAT7 configuration "+parms["KAT7Cfg"]
printMess(mess, logFile)
if debug:
pydoc.ttypager = pydoc.plainpager # don't page task input displays
mess = "Using Debug mode "
printMess(mess, logFile)
if check:
mess = "Only checking script"
printMess(mess, logFile)
# Log parameters
printMess("Parameter settings", logFile)
for p in parms:
mess = " "+p+": "+str(parms[p])
printMess(mess, logFile)
clist = []
for DCal in parms["DCals"]:
if DCal["Source"] not in clist:
clist.append(DCal["Source"])
for PCal in parms["PCals"]:
if PCal["Source"] not in clist:
clist.append(PCal["Source"])
for ACal in parms["ACals"]:
if ACal["Source"] not in clist:
clist.append(ACal["Source"])
if kwargs.get('targets') is not None:
targets = [targ.name for targ in katdata.catalogue if (targ.name not in clist) and (targ.name in kwargs.get('targets').split(','))]
else:
targets = [targ.name for targ in katdata.catalogue if (targ.name not in clist)]
refAnt = FetchObject(fileRoot+".refAnt.pickle")
# Save parameters to pickle jar, manifest
ParmsPicklefile = fileRoot+".Parms.pickle" # Where results saved
SaveObject(parms, ParmsPicklefile, True)
EVLAAddOutFile(os.path.basename(ParmsPicklefile), 'project', 'Processing parameters used' )
loadClass = dataClass
retCode = KATCalAvg (uv, "PREAVG", parms["seq"], parms["CalAvgTime"], err, \
flagVer=-1, doCalib=-1, gainUse=-1, doBand=-1, BPVer=-1, doPol=False, \
avgFreq=0, chAvg=1, BChan=1, EChan=0, doAuto=parms["doAuto"], Stokes=' ',\
BIF=parms["CABIF"], EIF=parms["CAEIF"], Compress=parms["Compress"], \
nThreads=nThreads, logfile=logFile, check=check, debug=debug)
if retCode!=0:
raise RuntimeError("Error in CalAvg")
uv.Zap(err)
# Get initially averaged data
if not check:
uv = UV.newPAUV("AIPS UV DATA", EVLAAIPSName(project), "PREAVG", \
disk, parms["seq"], True, err)
if err.isErr:
OErr.printErrMsg(err, "Error creating initial avg AIPS data")
# Hanning
parms["doHann"]=True
if parms["doHann"]:
# Set uv if not done
if uv==None and not check:
uv = UV.newPAUV("AIPS UV DATA", EVLAAIPSName(project), loadClass[0:6], disk, parms["seq"], True, err)
if err.isErr:
OErr.printErrMsg(err, "Error creating AIPS data")
uv = KATHann(uv, EVLAAIPSName(project), dataClass, disk, parms["seq"], err, \
doDescm=parms["doDescm"], flagVer=0, logfile=logFile, check=check, debug=debug)
#Halve channels after hanning.
parms["selChan"]=int(parms["selChan"]/2)
parms["BChDrop"]=int(parms["BChDrop"]/2)
parms["EChDrop"]=int(parms["EChDrop"]/2)
if uv==None and not check:
raise RuntimeError("Cannot Hann data ")
# Clear any old calibration/editing
if parms["doClearTab"]:
mess = "Clear previous calibration"
printMess(mess, logFile)
EVLAClearCal(uv, err, doGain=parms["doClearGain"], doFlag=parms["doClearFlag"], doBP=parms["doClearBP"], check=check)
OErr.printErrMsg(err, "Error resetting calibration")
# Quack to remove data from start and end of each scan
if parms["doQuack"]:
retCode = EVLAQuack (uv, err, begDrop=parms["quackBegDrop"], endDrop=parms["quackEndDrop"], \
Reason=parms["quackReason"], \
logfile=logFile, check=check, debug=debug)
if retCode!=0:
raise RuntimeError("Error Quacking data")
# Flag antennas shadowed by others?
if parms["doShad"]:
retCode = EVLAShadow (uv, err, shadBl=parms["shadBl"], \
logfile=logFile, check=check, debug=debug)
if retCode!=0:
raise RuntimeError("Error Shadow flagging data")
# Median window time editing, for RFI impulsive in time
if parms["doMednTD1"]:
mess = "Median window time editing, for RFI impulsive in time:"
printMess(mess, logFile)
retCode = EVLAMedianFlag (uv, clist, err, noScrat=noScrat, nThreads=nThreads, \
avgTime=parms["mednAvgTime"], avgFreq=parms["mednAvgFreq"], chAvg= parms["mednChAvg"], \
timeWind=parms["mednTimeWind"],flagVer=2, flagTab=2,flagSig=parms["mednSigma"], \
logfile=logFile, check=check, debug=False)
if retCode!=0:
raise RuntimeError("Error in MednFlag")
# Median window frequency editing, for RFI impulsive in frequency
if parms["doFD1"]:
mess = "Median window frequency editing, for RFI impulsive in frequency:"
printMess(mess, logFile)
retCode = EVLAAutoFlag (uv, clist, err, flagVer=2, flagTab=2, doCalib=-1, doBand=-1, \
timeAvg=parms["FD1TimeAvg"], \
doFD=True, FDmaxAmp=1.0e20, FDmaxV=1.0e20, FDwidMW=parms["FD1widMW"], \
FDmaxRMS=[1.0e20,0.1], FDmaxRes=parms["FD1maxRes"], \
FDmaxResBL= parms["FD1maxRes"], FDbaseSel=parms["FD1baseSel"],\
nThreads=nThreads, logfile=logFile, check=check, debug=debug)
if retCode!=0:
raise RuntimeError("Error in AutoFlag")
# Parallactic angle correction?
if parms["doPACor"]:
retCode = EVLAPACor(uv, err, \
logfile=logFile, check=check, debug=debug)
if retCode!=0:
raise RuntimeError("Error in Parallactic angle correction")
# Need to find a reference antenna? See if we have saved it?
if (parms["refAnt"]<=0):
refAnt = FetchObject(fileRoot+".refAnt.pickle")
if refAnt:
parms["refAnt"] = refAnt
# Use bandpass calibrator and center half of each spectrum
if parms["refAnt"]<=0:
mess = "Find best reference antenna: run Calib on BP Cal(s) "
printMess(mess, logFile)
parms["refAnt"] = EVLAGetRefAnt(uv, parms["BPCals"], err, flagVer=0, \
solInt=parms["bpsolint1"], nThreads=nThreads, \
logfile=logFile, check=check, debug=debug)
if err.isErr:
raise RuntimeError("Error finding reference antenna")
if parms["refAnts"][0]<=0:
parms["refAnts"][0] = parms["refAnt"]
mess = "Picked reference antenna "+str(parms["refAnt"])
printMess(mess, logFile)
# Save it
ParmsPicklefile = fileRoot+".Parms.pickle" # Where results saved
SaveObject(parms, ParmsPicklefile, True)
refAntPicklefile = fileRoot+".refAnt.pickle" # Where results saved
SaveObject(parms["refAnt"], refAntPicklefile, True)
# Plot Raw, edited data?
parms["doRawSpecPlot"]=False
parms["doSpecPlot"]=False
if parms["doRawSpecPlot"] and parms["plotSource"]:
mess = "Raw Spectral plot for: "+' '.join(parms["BPCal"])
printMess(mess, logFile)
plotFile = fileRoot+"_RawSpec.ps"
retCode = EVLASpectrum(uv, parms["BPCal"], parms["plotTime"], maxgap, plotFile, parms["refAnt"], err, \
Stokes=["RR","LL"], doband=-1, \
check=check, debug=debug, logfile=logFile )
if retCode!=0:
raise RuntimeError("Error in Plotting spectrum")
EVLAAddOutFile(plotFile, 'project', 'Pipeline log file' )
# delay calibration
if parms["doDelayCal"] and parms["DCals"] and not check:
plotFile = fileRoot+"_DelayCal.ps"
retCode = EVLADelayCal(uv, parms["DCals"], err, \
BChan=parms["delayBChan"], EChan=parms["delayEChan"], \
doCalib=2, flagVer=0, doBand=-1, \
solInt=parms["delaySolInt"], smoTime=parms["delaySmoo"], \
refAnts=[parms["refAnt"]], doTwo=parms["doTwo"],
doZeroPhs=parms["delayZeroPhs"], \
doPlot=parms["doSNPlot"], plotFile=plotFile, \
nThreads=nThreads, noScrat=noScrat, \
logfile=logFile, check=check, debug=debug)
if retCode!=0:
raise RuntimeError("Error in delay calibration")
# Plot corrected data?
if parms["doSpecPlot"] and parms["plotSource"]:
plotFile = fileRoot+"_DelaySpec.ps"
retCode = EVLASpectrum(uv, parms["BPCal"], parms["plotTime"], maxgap, \
plotFile, parms["refAnt"], err, \
Stokes=["RR","LL"], doband=-1, \
check=check, debug=debug, logfile=logFile )
if retCode!=0:
raise RuntimeError("Error in Plotting spectrum")
print(parms["bpBChan1"],parms["bpEChan1"],parms["bpBChan2"],parms["bpEChan2"],parms["bpChWid2"])
# Bandpass calibration
if parms["doBPCal"] and parms["BPCals"]:
retCode = KATBPCal(uv, parms["BPCals"], err, noScrat=noScrat, solInt1=parms["bpsolint1"], \
solInt2=parms["bpsolint2"], solMode=parms["bpsolMode"], \
BChan1=parms["bpBChan1"], EChan1=parms["bpEChan1"], \
BChan2=parms["bpBChan2"], EChan2=parms["bpEChan2"], ChWid2=parms["bpChWid2"], \
doCenter1=parms["bpDoCenter1"], refAnt=parms["refAnt"], \
UVRange=parms["bpUVRange"], doCalib=2, gainUse=0, flagVer=0, doPlot=False, \
nThreads=nThreads, logfile=logFile, check=check, debug=debug)
if retCode!=0:
raise RuntimeError("Error in Bandpass calibration")
# Plot corrected data?
if parms["doSpecPlot"] and parms["plotSource"]:
plotFile = fileRoot+"_BPSpec.ps"
retCode = EVLASpectrum(uv, parms["BPCal"], parms["plotTime"], maxgap, plotFile, \
parms["refAnt"], err, Stokes=["RR","LL"], doband=1, \
check=check, debug=debug, logfile=logFile )
if retCode!=0:
raise RuntimeError("Error in Plotting spectrum")
# Amp & phase Calibrate
if parms["doAmpPhaseCal"]:
plotFile = fileRoot+"_APCal.ps"
retCode = KATCalAP (uv, [], parms["ACals"], err, PCals=parms["PCals"],
doCalib=2, doBand=1, BPVer=1, flagVer=0, \
BChan=parms["ampBChan"], EChan=parms["ampEChan"], \
solInt=parms["solInt"], solSmo=parms["solSmo"], ampScalar=parms["ampScalar"], \
doAmpEdit=parms["doAmpEdit"], ampSigma=parms["ampSigma"], \
ampEditFG=parms["ampEditFG"], \
doPlot=parms["doSNPlot"], plotFile=plotFile, refAnt=parms["refAnt"], \
nThreads=nThreads, noScrat=noScrat, logfile=logFile, check=check, debug=debug)
#print parms["ACals"],parms["PCals"]
if retCode!=0:
raise RuntimeError("Error calibrating")
# More editing
if parms["doAutoFlag"]:
mess = "Post calibration editing:"
printMess(mess, logFile)
# if going to redo then only calibrators
if parms["doRecal"]:
# Only calibrators
clist = []
for DCal in parms["DCals"]:
if DCal["Source"] not in clist:
clist.append(DCal["Source"])
for PCal in parms["PCals"]:
if PCal["Source"] not in clist:
clist.append(PCal["Source"])
for ACal in parms["ACals"]:
if ACal["Source"] not in clist:
clist.append(ACal["Source"])
else:
clist = []
retCode = EVLAAutoFlag (uv, clist, err, flagVer=0, flagTab =2, \
doCalib=2, gainUse=0, doBand=1, BPVer=1, \
IClip=parms["IClip"], minAmp=parms["minAmp"], timeAvg=parms["timeAvg"], \
doFD=parms["doAFFD"], FDmaxAmp=parms["FDmaxAmp"], FDmaxV=parms["FDmaxV"], \
FDwidMW=parms["FDwidMW"], FDmaxRMS=parms["FDmaxRMS"], \
FDmaxRes=parms["FDmaxRes"], FDmaxResBL=parms["FDmaxResBL"], \
FDbaseSel=parms["FDbaseSel"], \
nThreads=nThreads, logfile=logFile, check=check, debug=debug)
if retCode!=0:
raise RuntimeError("Error in AutoFlag")
# Redo the calibration using new flagging?
if parms["doBPCal2"]==None:
parms["doBPCal2"] = parms["doBPCal"]
if parms["doDelayCal2"]==None:
parms["doDelayCal2"] = parms["doDelayCal2"]
if parms["doAmpPhaseCal2"]==None:
parms["doAmpPhaseCal2"] = parms["doAmpPhaseCal"]
if parms["doAutoFlag2"]==None:
parms["doAutoFlagCal2"] = parms["doAutoFlag"]
if parms["doRecal"]:
mess = "Redo calibration:"
printMess(mess, logFile)
EVLAClearCal(uv, err, doGain=True, doFlag=False, doBP=True, check=check, logfile=logFile)
OErr.printErrMsg(err, "Error resetting calibration")
# Parallactic angle correction?
if parms["doPACor"]:
retCode = EVLAPACor(uv, err, \
logfile=logFile, check=check, debug=debug)
if retCode!=0:
raise RuntimeError("Error in Parallactic angle correction")
# Delay recalibration
if parms["doDelayCal2"] and parms["DCals"] and not check:
plotFile = fileRoot+"_DelayCal2.ps"
retCode = EVLADelayCal(uv, parms["DCals"], err, \
BChan=parms["delayBChan"], EChan=parms["delayEChan"], \
doCalib=2, flagVer=0, doBand=-1, \
solInt=parms["delaySolInt"], smoTime=parms["delaySmoo"], \
refAnts=[parms["refAnt"]], doTwo=parms["doTwo"], \
doZeroPhs=parms["delayZeroPhs"], \
doPlot=parms["doSNPlot"], plotFile=plotFile, \
nThreads=nThreads, noScrat=noScrat, \
logfile=logFile, check=check, debug=debug)
if retCode!=0:
raise RuntimeError("Error in delay calibration")
# Plot corrected data?
if parms["doSpecPlot"] and parms["plotSource"]:
plotFile = fileRoot+"_DelaySpec2.ps"
retCode = EVLASpectrum(uv, parms["BPCal"], parms["plotTime"], maxgap, plotFile, parms["refAnt"], err, \
Stokes=["RR","LL"], doband=-1, \
check=check, debug=debug, logfile=logFile )
if retCode!=0:
raise RuntimeError("Error in Plotting spectrum")
# Bandpass calibration
if parms["doBPCal2"] and parms["BPCals"]:
retCode = KATBPCal(uv, parms["BPCals"], err, noScrat=noScrat, solInt1=parms["bpsolint1"], \
solInt2=parms["bpsolint2"], solMode=parms["bpsolMode"], \
BChan1=parms["bpBChan1"], EChan1=parms["bpEChan1"], \
BChan2=parms["bpBChan2"], EChan2=parms["bpEChan2"], ChWid2=parms["bpChWid2"], \
doCenter1=parms["bpDoCenter1"], refAnt=parms["refAnt"], \
UVRange=parms["bpUVRange"], doCalib=2, gainUse=0, flagVer=0, doPlot=False, \
nThreads=nThreads, logfile=logFile, check=check, debug=debug)
if retCode!=0:
raise RuntimeError("Error in Bandpass calibration")
# Plot corrected data?
if parms["doSpecPlot"] and parms["plotSource"]:
plotFile = fileRoot+"_BPSpec2.ps"
retCode = EVLASpectrum(uv, parms["BPCal"], parms["plotTime"], maxgap, plotFile, parms["refAnt"], err, \
Stokes=["RR","LL"], doband=1, \
check=check, debug=debug, logfile=logFile )
if retCode!=0:
raise RuntimeError("Error in Plotting spectrum")
# Amp & phase Recalibrate
if parms["doAmpPhaseCal2"]:
plotFile = fileRoot+"_APCal2.ps"
retCode = KATCalAP (uv, [], parms["ACals"], err, PCals=parms["PCals"], \
doCalib=2, doBand=2, BPVer=1, flagVer=0, \
BChan=parms["ampBChan"], EChan=parms["ampEChan"], \
solInt=parms["solInt"], solSmo=parms["solSmo"], ampScalar=parms["ampScalar"], \
doAmpEdit=True, ampSigma=parms["ampSigma"], \
ampEditFG=parms["ampEditFG"], \
doPlot=parms["doSNPlot"], plotFile=plotFile, refAnt=parms["refAnt"], \
noScrat=noScrat, nThreads=nThreads, logfile=logFile, check=check, debug=debug)
if retCode!=0:
raise RuntimeError("Error calibrating")
# More editing
if parms["doAutoFlag2"]:
mess = "Post recalibration editing:"
printMess(mess, logFile)
retCode = EVLAAutoFlag (uv, [], err, flagVer=0, flagTab=2, \
doCalib=2, gainUse=0, doBand=1, BPVer=1, \
IClip=parms["IClip"], minAmp=parms["minAmp"], timeAvg=parms["timeAvg"], \
doFD=parms["doAFFD"], FDmaxAmp=parms["FDmaxAmp"], FDmaxV=parms["FDmaxV"], \
FDwidMW=parms["FDwidMW"], FDmaxRMS=parms["FDmaxRMS"], \
FDmaxRes=parms["FDmaxRes"], FDmaxResBL= parms["FDmaxResBL"], \
FDbaseSel=parms["FDbaseSel"], \
nThreads=nThreads, logfile=logFile, check=check, debug=debug)
if retCode!=0:
raise RuntimeError("Error in AutoFlag")
# end recal
# Calibrate and average data
if parms["doCalAvg"]:
retCode = KATCalAvg (uv, avgClass, parms["seq"], parms["CalAvgTime"], err, \
flagVer=2, doCalib=2, gainUse=0, doBand=1, BPVer=1, doPol=False, \
avgFreq=parms["avgFreq"], chAvg=parms["chAvg"], \
BChan=1, EChan=parms["selChan"] - 1, doAuto=parms["doAuto"], \
BIF=parms["CABIF"], EIF=parms["CAEIF"], Compress=parms["Compress"], \
nThreads=nThreads, logfile=logFile, check=check, debug=debug)
if retCode!=0:
raise RuntimeError("Error in CalAvg")
# Get calibrated/averaged data
if not check:
uv = UV.newPAUV("AIPS UV DATA", EVLAAIPSName(project), avgClass[0:6], \
disk, parms["seq"], True, err)
if err.isErr:
OErr.printErrMsg(err, "Error creating cal/avg AIPS data")
KATUVFITS(uv, 'preimage.uvfits', 0, err, exclude=["AIPS HI", "AIPS SL", "AIPS PL"], include=["AIPS AN", "AIPS FQ"], compress=parms["Compress"], logfile=logFile)
KATUVFITab(uv, 'preimage.uvtab', 0, err)
# XClip
if parms["XClip"] and parms["XClip"]>0.0:
mess = "Cross Pol clipping:"
printMess(mess, logFile)
retCode = EVLAAutoFlag (uv, [], err, flagVer=-1, flagTab=1, \
doCalib=2, gainUse=0, doBand=-1, maxBad=1.0, \
XClip=parms["XClip"], timeAvg=1./60., \
nThreads=nThreads, logfile=logFile, check=check, debug=debug)
if retCode!=0:
raise RuntimeError("Error in AutoFlag")
# R-L delay calibration cal if needed,
if parms["doRLDelay"] and parms["RLDCal"][0][0]!=None:
if parms["rlrefAnt"]<=0:
parms["rlrefAnt"] = parms["refAnt"]
# parms["rlDoBand"] if before average, BPVer=parms["rlBPVer"],
retCode = EVLARLDelay(uv, err,\
RLDCal=parms["RLDCal"], BChan=parms["rlBChan"], \
EChan=parms["rlEChan"], UVRange=parms["rlUVRange"], \
soucode=parms["rlCalCode"], doCalib=parms["rlDoCal"], gainUse=parms["rlgainUse"], \
timerange=parms["rltimerange"], \
# NOT HERE doBand=parms["rlDoBand"], BPVer=parms["rlBPVer"], \
flagVer=parms["rlflagVer"], \
refAnt=parms["rlrefAnt"], doPol=False, \
nThreads=nThreads, noScrat=noScrat, logfile=logFile, \
check=check, debug=debug)
if retCode!=0:
raise RuntimeError("Error in R-L delay calibration")
# Polarization calibration
if parms["doPolCal"]:
if parms["PCRefAnt"]<=0:
parms["PCRefAnt"] = parms["refAnt"]
retCode = EVLAPolCal(uv, parms["PCInsCals"], err, \
doCalib=2, gainUse=0, doBand=-1, flagVer=0, \
fixPoln=parms["PCFixPoln"], pmodel=parms["PCpmodel"], avgIF=parms["PCAvgIF"], \
solInt=parms["PCSolInt"], refAnt=parms["PCRefAnt"], solType=parms["PCSolType"], \
ChInc=parms["PCChInc"], ChWid=parms["PCChWid"], \
nThreads=nThreads, check=check, debug=debug, noScrat=noScrat, logfile=logFile)
if retCode!=0 and (not check):
raise RuntimeError("Error in polarization calibration: "+str(retCode))
# end poln cal.
# R-L phase calibration cal., creates new BP table
if parms["doRLCal"] and parms["RLDCal"][0][0]!=None:
plotFile = fileRoot+"_RLSpec2.ps"
if parms["rlrefAnt"]<=0:
parms["rlrefAnt"] = parms["refAnt"]
retCode = EVLARLCal(uv, err,\
RLDCal=parms["RLDCal"], BChan=parms["rlBChan"],
EChan=parms["rlEChan"], UVRange=parms["rlUVRange"], \
ChWid2=parms["rlChWid"], solInt1=parms["rlsolint1"], solInt2=parms["rlsolint2"], \
RLPCal=parms["RLPCal"], RLPhase=parms["RLPhase"], \
RM=parms["RLRM"], CleanRad=parms["rlCleanRad"], \
calcode=parms["rlCalCode"], doCalib=parms["rlDoCal"], gainUse=parms["rlgainUse"], \
timerange=parms["rltimerange"], FOV=parms["rlFOV"], \
doBand=-1, BPVer=1, flagVer=parms["rlflagVer"], \
refAnt=parms["rlrefAnt"], doPol=parms["doPol"], PDVer=parms["PDVer"], \
doPlot=parms["doSpecPlot"], plotFile=plotFile, \
nThreads=nThreads, noScrat=noScrat, logfile=logFile, \
check=check, debug=debug)
if retCode!=0:
raise RuntimeError("Error in RL phase spectrum calibration")
# VClip
if parms["VClip"] and parms["VClip"]>0.0:
mess = "VPol clipping:"
printMess(mess, logFile)
retCode = EVLAAutoFlag (uv, [], err, flagVer=-1, flagTab=1, \
doCalib=2, gainUse=0, doBand=-1, \
VClip=parms["VClip"], timeAvg=parms["timeAvg"], \
nThreads=nThreads, logfile=logFile, check=check, debug=debug)
if retCode!=0:
raise RuntimeError("Error in AutoFlag VClip")
# Plot corrected data?
parms["doSpecPlot"]=True
if parms["doSpecPlot"] and parms["plotSource"]:
plotFile = fileRoot+"_Spec.ps"
retCode = EVLASpectrum(uv, parms["BPCal"], parms["plotTime"], maxgap, \
plotFile, parms["refAnt"], err, \
Stokes=["I"], doband=-1, \
check=check, debug=debug, logfile=logFile )
if retCode!=0:
raise RuntimeError("Error in Plotting spectrum")
# Image targets
if parms["doImage"]:
# If targets not specified, image all
if len(parms["targets"])<=0:
slist = EVLAAllSource(uv,err,logfile=logFile,check=check,debug=debug)
else:
slist = targets
slist=targets
KATImageTargets (uv, err, Sources=slist, seq=parms["seq"], sclass=outIClass, OutlierArea=parms["outlierArea"],\
doCalib=-1, doBand=-1, flagVer=-1, doPol=parms["doPol"], PDVer=parms["PDVer"], \
Stokes=parms["Stokes"], FOV=parms["FOV"], Robust=parms["Robust"], Niter=parms["Niter"], \
CleanRad=parms["CleanRad"], minFlux=parms["minFlux"], OutlierSize=parms["OutlierSize"], \
xCells=parms["xCells"], yCells=parms["yCells"], Reuse=parms["Reuse"], minPatch=parms["minPatch"], \
maxPSCLoop=parms["maxPSCLoop"], minFluxPSC=parms["minFluxPSC"], noNeg=parms["noNeg"], \
solPInt=parms["solPInt"], solPMode=parms["solPMode"], solPType=parms["solPType"], \
maxASCLoop=parms["maxASCLoop"], minFluxASC=parms["minFluxASC"], nx=parms["nx"], ny=parms["ny"], \
solAInt=parms["solAInt"], solAMode=parms["solAMode"], solAType=parms["solAType"], \
avgPol=parms["avgPol"], avgIF=parms["avgIF"], minSNR = parms["minSNR"], refAnt=parms["refAnt"], \
do3D=parms["do3D"], BLFact=parms["BLFact"], BLchAvg=parms["BLchAvg"], \
doMB=parms["doMB"], norder=parms["MBnorder"], maxFBW=parms["MBmaxFBW"], \
PBCor=parms["PBCor"],antSize=parms["antSize"], autoCen=parms["autoCen"], \
nTaper=parms["nTaper"], Tapers=parms["Tapers"], sefd=sefd, \
nThreads=nThreads, noScrat=noScrat, logfile=logFile, check=check, debug=False)
# End image
# Get report on sources
if parms["doReport"]:
# If targets not specified, do all
if len(parms["targets"])<=0:
slist = EVLAAllSource(uv,err,logfile=logFile,check=check,debug=debug)
else:
slist = parms["targets"]
Report = EVLAReportTargets(uv, err, Sources=slist, seq=parms["seq"], sclass=outIClass, \
Stokes=parms["Stokes"], logfile=logFile, check=check, debug=debug)
# Save to pickle jar
ReportPicklefile = fileRoot+"_Report.pickle" # Where results saved
SaveObject(Report, ReportPicklefile, True)
# Write results, cleanup
# Save cal/average UV data?
if parms["doSaveUV"] and (not check):
Aname = EVLAAIPSName(project)
cno = AIPSDir.PTestCNO(disk, user, Aname, avgClass[0:6], "UV", parms["seq"], err)
if cno>0:
uvt = UV.newPAUV("AIPS CAL UV DATA", Aname, avgClass, disk, parms["seq"], True, err)
filename = fileRoot+"_Cal.uvtab"
KATUVFITS (uv, filename, 0, err, exclude=["AIPS HI", "AIPS SL", "AIPS PL"], include=["AIPS AN", "AIPS FQ"], compress=parms["Compress"], logfile=logFile)
EVLAAddOutFile(os.path.basename(filename), 'project', "Calibrated Averaged UV data" )
# Save list of output files
EVLASaveOutFiles(manifestfile)
del uvt
# Imaging results
# If targets not specified, save all
if len(parms["targets"])<=0:
slist = EVLAAllSource(uv,err,logfile=logFile,check=check,debug=debug)
else:
slist = parms["targets"]
for target in slist:
if parms["doSaveImg"] and (not check):
for s in parms["Stokes"]:
oclass = s+outIClass[1:]
outname = target
# Test if image exists
cno = AIPSDir.PTestCNO(disk, user, outname, oclass, "MA", parms["seq"], err)
#print cno
if cno <= 0 :
continue
x = Image.newPAImage("out", outname, oclass, disk, parms["seq"], True, err)
outfilefits = fileRoot+'_'+target+"."+oclass+".fits"
xf = KATImFITS(x, outfilefits, 0, err, logfile=logFile)
x = Image.newPAImage("out", outname, oclass, disk, parms["seq"], True, err)
outfile = fileRoot+'_'+target+"."+oclass+".fittab.fits"
xf = EVLAImFITS (x, outfile, 0, err, logfile=logFile)
EVLAAddOutFile(outfile, target, 'Image of '+ target)
# Statistics
zz=imstat(x, err, logfile=logFile)
# Make a Jpeg image
FITS2jpeg.fits2jpeg(outfilefits,chans=1,contrast=0.05,cmap='jet',area=0.7)
EVLAAddOutFile(outfile.replace('.fits','.jpeg'), target, 'Jpeg image of '+ target)
# end writing loop
# Save list of output files
EVLASaveOutFiles(manifestfile)
OErr.printErrMsg(err, "Writing output")
# Contour plots
if parms["doKntrPlots"]:
mess = "INFO --> Contour plots (doKntrPlots)"
printMess(mess, logFile)
EVLAKntrPlots( err, imName=parms["targets"], project=fileRoot,
disk=disk, debug=debug )
# Save list of output files
EVLASaveOutFiles(manifestfile)
elif debug:
mess = "Not creating contour plots ( doKntrPlots = "+str(parms["doKntrPlots"])+ " )"
printMess(mess, logFile)
# Source uv plane diagnostic plots
if parms["doDiagPlots"]:
mess = "INFO --> Diagnostic plots (doDiagPlots)"
printMess(mess, logFile)
# Get the highest number avgClass catalog file
Aname = EVLAAIPSName( project )
uvc = None
if not check:
uvname = project+"_Cal"
uvc = UV.newPAUV(uvname, Aname, avgClass, disk, parms["seq"], True, err)
EVLADiagPlots( uvc, err, cleanUp=parms["doCleanup"], \
project=fileRoot, \
logfile=logFile, check=check, debug=debug )
# Save list of output files
EVLASaveOutFiles(manifestfile)
elif debug:
mess = "Not creating diagnostic plots ( doDiagPlots = "+str(parms["doDiagPlots"])+ " )"
printMess(mess, logFile)
# Save metadata
srcMetadata = None
projMetadata = None
if parms["doMetadata"]:
mess = "INFO --> Save metadata (doMetadata)"
printMess(mess, logFile)
uvc = None
if not uvc:
# Get calibrated/averaged data
Aname = EVLAAIPSName(project)
uvname = project+"_Cal"
uvc = UV.newPAUV(uvname, Aname, avgClass, disk, parms["seq"], True, err)
if err.isErr:
OErr.printErrMsg(err, "Error creating cal/avg AIPS data")
# Get source metadata; save to pickle file
srcMetadata = EVLASrcMetadata( uvc, err, Sources=parms["targets"], seq=parms["seq"], \
sclass=outIClass, Stokes=parms["Stokes"],\
logfile=logFile, check=check, debug=debug )
picklefile = fileRoot+".SrcReport.pickle"
SaveObject( srcMetadata, picklefile, True )
EVLAAddOutFile(os.path.basename(picklefile), 'project', 'All source metadata' )
# Get project metadata; save to pickle file
projMetadata = KATProjMetadata( uvc, AIPS_VERSION, err, \
PCals=parms["PCals"], ACals=parms["ACals"], \
BPCals=parms["BPCals"], DCals=parms["DCals"], \
project = project, band = band, \
dataInUVF = parms["archRoot"], archFileID = fileRoot )
picklefile = fileRoot+".ProjReport.pickle"
SaveObject(projMetadata, picklefile, True)
EVLAAddOutFile(os.path.basename(picklefile), 'project', 'Project metadata' )
else:
# Fetch from pickle jar
picklefile = fileRoot+".SrcReport.pickle"
srcMetadata = FetchObject(picklefile)
picklefile = fileRoot+".ProjReport.pickle"
projMetadata = FetchObject(picklefile)
# Write report
if parms["doHTML"]:
mess = "INFO --> Write HTML report (doHTML)"
printMess(mess, logFile)
KATHTMLReport( projMetadata, srcMetadata, \
outfile=fileRoot+"_report.html", \
logFile=logFile )
# Write VOTable
if parms["doVOTable"]:
mess = "INFO --> Write VOTable (doVOTable)"
printMess(mess, logFile)
EVLAAddOutFile( 'VOTable.xml', 'project', 'VOTable report' )
EVLAWriteVOTable( projMetadata, srcMetadata, filename=fileRoot+'_VOTable.xml' )
# Save list of output files
EVLASaveOutFiles(manifestfile)
# Cleanup - delete AIPS files
if parms["doCleanup"] and (not check):
mess = "INFO --> Clean up (doCleanup)"
printMess(mess, logFile)
# Delete target images
# How many Stokes images
nstok = len(parms["Stokes"])
for istok in range(0,nstok):
oclass = parms["Stokes"][istok:istok+1]+outIClass[1:]
AllDest(err, disk=disk,Aseq=parms["seq"],Aclass=oclass)
# Delete initial UV data
Aname = EVLAAIPSName(project)
# Test if data exists
cno = AIPSDir.PTestCNO(disk, user, Aname, dataClass[0:6], "UV", parms["seq"], err)
if cno>0:
uvt = UV.newPAUV("AIPS RAW UV DATA", Aname, dataClass[0:6], disk, parms["seq"], True, err)
uvt.Zap(err)
del uvt
if err.isErr:
OErr.printErrMsg(err, "Error deleting raw AIPS data")
# Zap calibrated/averaged data
# Test if data exists
cno = AIPSDir.PTestCNO(disk, user, Aname, avgClass[0:6], "UV", parms["seq"], err)
if cno>0:
uvt = UV.newPAUV("AIPS CAL UV DATA", Aname, avgClass[0:6], disk, parms["seq"], True, err)
uvt.Zap(err)
del uvt
if err.isErr:
OErr.printErrMsg(err, "Error deleting cal/avg AIPS data")
# Zap UnHanned data if present
loadClass = "Raw"
# Test if image exists
cno = AIPSDir.PTestCNO(disk, user, Aname, loadClass[0:6], "UV", parms["seq"], err)
if cno>0:
uvt = UV.newPAUV("AIPS CAL UV DATA", Aname, loadClass[0:6], disk, parms["seq"], True, err)
uvt.Zap(err)
del uvt
if err.isErr:
OErr.printErrMsg(err, "Error deleting cal/avg AIPS data")
OErr.printErrMsg(err, "Writing output/cleanup")
# Delete AIPS scratch DA00 and disk
if os.path.exists(os.environ['DA00']): shutil.rmtree(os.environ['DA00'])
for disk in ObitTalkUtil.AIPSDir.AIPSdisks:
if os.path.exists(disk): shutil.rmtree(disk)
# Shutdown
mess = "Finished project "+parms["project"]+ \
" AIPS user no. "+str(AIPS.userno)
printMess(mess, logFile)
OErr.printErr(err)
OSystem.Shutdown(ObitSys)
