def calibrate(msname='$MS',
lsmname='$LSM',
column='$COLUMN',
do_dE=False,
args=[],
options={},
**kw):
""" Calibrate MS """
msname, lsmname, column, tdlconf, tdlsec = interpolate_locals(
'msname lsmname '
'column tdlconf tdlsec')
v.MS = msname
v.LSM = lsmname
args = ["${ms.MS_TDL} ${lsm.LSM_TDL}"] + list(args)
options.update(
dict(diffgain_plot_prefix=None,
gain_plot_prefix=None,
ifrgain_plot_prefix=None))
options["pybeams_fits.filename_pattern"] = BEAM_PATTERN
if do_dE:
""" add dE opts into options dict"""
options.update(
dict(diffgains=True, stefcal_reset_all=True, diffgain_plot=True))
stefcal.stefcal(msname, options=options, args=args, **kw)
def runcal(goto_step=1):
## initial calibration
if goto_step > 1:
info("########## restarting calibration from step $goto_step")
# Calibration step -- this is just a user-defined label used in filenames (etc. "blahblah_s1"), which serves to keep the output from each step
# of a pipeline separate. If this is numeric, then functions such as stefcal.stefcal() will increment it automatically first thing. Otherwise you can set
# it yourself to some more fancy label. Here we also provide a way to hop to particular step via goto_step
v.STEP = goto_step - 1
if goto_step < 2:
# set the superglobal LSM
v.LSM = LSM0
info("########## solving for G with initial LSM")
# no w-proj for dirty map to save time
stefcal.stefcal(stefcal_reset_ifr_gains=True,
dirty=dict(wprojplanes=0),
restore=True)
info("########## running source finder and updating model")
## now run pybdsm on restored image, output LSM will be given by variable cal.PYBDSM_OUTPUT
lsm.pybdsm_search(threshold=7)
### merge new sources into sky model, give it a new name ($LSM1)
lsm.tigger_convert("$LSM -a ${lsm.PYBDSM_OUTPUT} $LSM1 --rename -f")
if goto_step < 3:
info("########## solving for G with updated LSM (initial+pybdsm)")
v.LSM = LSM1
stefcal.stefcal(dirty=dict(wprojplanes=0))
if goto_step < 4:
info("########## re-solving for G to apply IFR solutions")
stefcal.stefcal(dirty=dict(wprojplanes=0), restore=True)
info("########## adding clean components to LSM")
CCMODEL = II("ccmodel-ddid${ms.DDID}.fits")
# note the per-style variable interpolation done by the II() function
ff = pyfits.open(imager.MODEL_IMAGE)
dd = ff[0].data
dd *= 1.0769 # scale up to compensate for selfcal flux suppression
# dd[dd<0] = 0; # remove negative components
ff.writeto(CCMODEL, clobber=True)
# add model image to LSM
lsm.tigger_convert("$LSM $LSM2 --add-brick=ccmodel:$CCMODEL:2 -f")
if goto_step < 5:
info("########## solving for G with updated LSM (inital+pybdsm+cc)")
v.LSM = LSM2
stefcal.stefcal(dirty=dict(wprojplanes=0))
if goto_step < 6:
info("########## running DD solutions")
v.LSM = LSM2
# now, set dE tags on sources
lsm.transfer_tags(LSMREF, LSM, tags="dE", tolerance=45 * ARCSEC)
# make final image
stefcal.stefcal(dirty=dict(wprojplanes=0),
diffgains=True,
restore=True,
label="dE")
def runcal (goto_step=1):
## initial calibration
if goto_step > 1:
info("########## restarting calibration from step $goto_step");
# Calibration step -- this is just a user-defined label used in filenames (etc. "blahblah_s1"), which serves to keep the output from each step
# of a pipeline separate. If this is numeric, then functions such as stefcal.stefcal() will increment it automatically first thing. Otherwise you can set
# it yourself to some more fancy label. Here we also provide a way to hop to particular step via goto_step
v.STEP = goto_step-1;
if goto_step < 2:
# set the superglobal LSM
v.LSM = LSM0
info("########## solving for G with initial LSM");
# no w-proj for dirty map to save time
stefcal.stefcal(stefcal_reset_ifr_gains=True,dirty=dict(wprojplanes=0),restore=True);
info("########## running source finder and updating model");
## now run pybdsm on restored image, output LSM will be given by variable cal.PYBDSM_OUTPUT
lsm.pybdsm_search(threshold=7);
### merge new sources into sky model, give it a new name ($LSM1)
lsm.tigger_convert("$LSM -a ${lsm.PYBDSM_OUTPUT} $LSM1 --rename -f");
if goto_step < 3:
info("########## solving for G with updated LSM (initial+pybdsm)");
v.LSM = LSM1
stefcal.stefcal(dirty=dict(wprojplanes=0));
if goto_step < 4:
info("########## re-solving for G to apply IFR solutions");
stefcal.stefcal(dirty=dict(wprojplanes=0),restore=True);
info("########## adding clean components to LSM");
CCMODEL = II("ccmodel-ddid${ms.DDID}.fits"); # note the per-style variable interpolation done by the II() function
ff = pyfits.open(imager.MODEL_IMAGE);
dd = ff[0].data;
dd *= 1.0769 # scale up to compensate for selfcal flux suppression
# dd[dd<0] = 0; # remove negative components
ff.writeto(CCMODEL,clobber=True);
# add model image to LSM
lsm.tigger_convert("$LSM $LSM2 --add-brick=ccmodel:$CCMODEL:2 -f");
if goto_step < 5:
info("########## solving for G with updated LSM (inital+pybdsm+cc)");
v.LSM = LSM2
stefcal.stefcal(dirty=dict(wprojplanes=0));
if goto_step < 6:
info("########## running DD solutions");
v.LSM = LSM2
# now, set dE tags on sources
lsm.transfer_tags(LSMREF,LSM,tags="dE",tolerance=45*ARCSEC);
# make final image
stefcal.stefcal(dirty=dict(wprojplanes=0),diffgains=True,restore=True,label="dE");
def c_cal (goto_step=1):
"""Calibration for C-config data"""
## initial calibration
if goto_step > 1:
info("########## restarting calibration from step $goto_step");
# Calibration step -- this is just a user-defined label used in filenames (etc. "blahblah_s1"), which serves to keep the output from each step
# of a pipeline separate. If this is numeric, then functions such as stefcal.stefcal() will increment it automatically first thing. Otherwise you can set
# it yourself to some more fancy label. Here we also provide a way to hop to particular step via goto_step
v.STEP = goto_step-1;
if goto_step < 2:
# set the superglobal LSM
v.LSM = LSM0
info("########## solving for G with initial LSM");
# no w-proj for dirty map to save time
stefcal.stefcal(stefcal_reset_all=True,
dirty=dict(wprojplanes=0,npix=NPIX),
restore=dict(npix=NPIX,threshold=CLEAN_THRESH[0],wprojplanes=128));
info("########## running source finder and updating model");
## now run pybdsm on restored image, output LSM will be given by variable cal.PYBDSM_OUTPUT
### NB: select on radius to exclude any artefacts picked up around 3C147 itself
lsm.pybdsm_search(thresh_pix=THRESH_PIX,thresh_isl=THRESH_ISL,select="r.gt.30s");
### merge new sources into sky model, give it a new name ($LSM1)
lsm.tigger_convert("$LSM -a ${lsm.PYBDSM_OUTPUT} $LSM1 --rename -f");
if goto_step < 3:
info("########## solving for G+dE with updated LSM (initial+pybdsm)");
v.LSM = LSM1
# now, set dE tags on sources
lsm.transfer_tags(LSMREF,LSM,tags="dE",tolerance=45*ARCSEC);
stefcal.stefcal(stefcal_reset_all=True,diffgains=True,dirty=dict(wprojplanes=0,npix=NPIX));
if goto_step < 4:
info("########## re-solving for G to apply IFR solutions");
v.LSM = LSM1
stefcal.stefcal(diffgains=True,diffgain_apply_only=True,
dirty=dict(wprojplanes=0,npix=NPIX),
restore=dict(npix=NPIX,threshold=CLEAN_THRESH[1]));
info("########## adding clean components to LSM");
ff = pyfits.open(imager.MODEL_IMAGE);
dd = ff[0].data;
dd *= 1.0769 # scale up to compensate for selfcal flux suppression
# dd[dd<0] = 0; # remove negative components
ff.writeto(LSM_CCMODEL,clobber=True);
# add model image to LSM
lsm.tigger_convert("$LSM $LSM2 --add-brick=ccmodel:$LSM_CCMODEL:2 -f");
if goto_step < 5:
info("########## re-running DD solutions");
v.LSM = LSM2
# make final image
stefcal.stefcal(dirty=dict(wprojplanes=0,npix=NPIX),diffgains=True,
restore=dict(npix=NPIX,threshold=CLEAN_THRESH[2]),
label="dE");
# make per-channel cube
makecube(NPIX);
# make noise images
makenoise();
def calibrate_DI(msname='$MS',
lsmname='$LSM',
tdlcon='$TDLCONF',
tdlsec='${stefcal.STEFCAL_SECTION}',
timeint=None,
freqint=None,
smooth=None,
column='$COLUMN',
do_dE=False,
options={},
args=[],
**kw):
""" Calibrate MS """
msname, lsmname, column, tdlcon, tdlsec = \
interpolate_locals('msname lsmname column tdlcon tdlsec')
v.MS = msname
v.LSM = lsmname
args = ["${ms.MS_TDL} ${lsm.LSM_TDL}"] + list(args)
# Direction independent Calibration (Gain and Bandpass calibration)
if timeint is None:
timeint = DI_TIME_INTERVAL
if freqint is None:
freqint = DI_FREQ_INTERVAL
if smooth is None:
smooth = DI_SMOOTHING
if smooth:
kw['stefcal_gain.freqint'] = kw['stefcal_gain.timeint'] = 0
kw['stefcal_gain.freqsmooth'] = freqint
kw['stefcal_gain.timesmooth'] = timeint
else:
kw['stefcal_gain.freqsmooth'] = kw['stefcal_gain.timesmooth'] = 0
kw['stefcal_gain.freqint'] = freqint
kw['stefcal_gain.timeint'] = timeint
stefcal.stefcal(msname,
section=tdlsec,
diffgains=False,
output="CORR_RES",
restore=dict(niter=1000),
restore_lsm=False,
apply_only=False,
flag_threshold=(1, .5),
options=options,
args=args,
**kw)
def calibrate(msname='$MS', lsmname='$LSM',
column='$COLUMN', do_dE=False, args=[], options={}, **kw):
""" Calibrate MS """
msname,lsmname,column,tdlconf,tdlsec = interpolate_locals('msname lsmname '
'column tdlconf tdlsec')
v.MS = msname
v.LSM = lsmname
args = ["${ms.MS_TDL} ${lsm.LSM_TDL}"] + list(args)
options.update(dict(diffgain_plot_prefix=None,gain_plot_prefix=None,ifrgain_plot_prefix=None))
options["pybeams_fits.filename_pattern"] = BEAM_PATTERN
if do_dE:
""" add dE opts into options dict"""
options.update(dict(diffgains=True,stefcal_reset_all=True,diffgain_plot=True))
stefcal.stefcal(msname, options=options, args=args, **kw)
def calibrate(msname='$MS', lsmname='$LSM', tdlsec='$CALSEC',
column='$COLUMN', do_dE=False, args=[],**kw):
""" Calibrate MS """
msname, lsmname, column, tdlsec = \
interpolate_locals('msname lsmname column tdlsec')
v.MS = msname
v.LSM = lsmname
args = ["${ms.MS_TDL} ${lsm.LSM_TDL}"] + list(args)
v.LSM = lsmname
options = {}
if do_dE:
""" add dE opts into options dict"""
options.update(dict(diffgains=True))
options.update(kw)
stefcal.stefcal(msname,section=tdlsec,options=options,args=args)
def calibrate(msname='$MS',
lsmname='$LSM',
tdlsec='$CALSEC',
column='$COLUMN',
do_dE=False,
args=[],
**kw):
""" Calibrate MS """
msname, lsmname, column, tdlsec = \
interpolate_locals('msname lsmname column tdlsec')
v.MS = msname
v.LSM = lsmname
args = ["${ms.MS_TDL} ${lsm.LSM_TDL}"] + list(args)
v.LSM = lsmname
options = {}
if do_dE:
""" add dE opts into options dict"""
options.update(dict(diffgains=True))
options.update(kw)
stefcal.stefcal(msname, section=tdlsec, options=options, args=args)
def process(self, time, exposure):
# This function must be implemented.
# First get the data arrays needed by this step.
time0 = systime()
self.getData(self.itsData)
self.getModelData(self.itsModelData)
self.getFlags(self.itsFlags)
self.getWeights(self.itsWeights)
self.getUVW(self.itsUVW)
self.itsTimeFill += systime() - time0
# Process the data.
#print "process tPythonStep", time-4.47203e9, exposure, self.itsData.sum(), self.itsFlags.sum(), self.itsWeights.sum(), self.itsUVW.sum()
#print self.itsData.shape
# Execute the next step in the DPPP pipeline. TIME,UVW are changed.
time0 = systime()
flaggedStations = self.findFlaggedStations(self.itsFlags)
self.itsTimeFlag += systime() - time0
if len(flaggedStations) > 0:
raise Exception(
'Yikes, cannot handle totally flagged station, which is the case for station '
+ str(flaggedStations[0]) + ' (' +
self.itsInfo['AntNames'][flaggedStations[0]] + ')')
time0 = systime()
self.fillDataCubes(self.itsData, self.itsModelData, self.itsFlags,
self.itsWeights)
self.itsTimeReorder += systime() - time0
self.itsTimeSlot += 1
time0 = systime()
self.itsSols.append(stefcal(self.itsDataCube, self.itsModelDataCube))
self.itsTimeSolve += systime() - time0
return self.processNext({'DATA': self.itsData})
def calibrate_beam(msname='$MS',
lsmname='$LSM0',
tdlcon='$TDLCONF',
timeint=None,
freqint=None,
smooth=None,
tdlsec='${stefcal.STEFCAL_SECTION3}',
column='$COLUMN',
do_dE=True,
options={},
args=[],
**kw):
msname, lsmname, column, tdlcon, tdlsec = \
interpolate_locals('msname lsmname column tdlcon tdlsec')
v.MS = msname
v.LSM = lsmname
args = ["${ms.MS_TDL} ${lsm.LSM_TDL}"] + list(args)
options[
'pybeams_fits.filename_pattern'] = './kat_beams/KAT7_$$(XY)-chan-304-allfreq-$$(realimag).fits'
options['critical_flag_threshold'] = 50.0
options['me.e_enable'] = 1.0
options['pybeams_fits.l_axis'] = FITS_L_AXIS
options['pybeams_fits.m_axis'] = FITS_M_AXIS
options['pybeams_fits.ampl_interpolation'] = 0.0
options['pybeams_fits.l_beam_offset'] = 0.0
options['pybeams_fits.m_beam_offset'] = 0.0
options['pybeams_fits.missing_is_null'] = 1
options['pybeams_fits.normalize_gains'] = 0.0
options['pybeams_fits.verbose_level'] = None
options['pybeams_fits.sky_rotation'] = 1
options['pybeams_fits.spline_order'] = 3
options['pybeams_fits.normalize_gains'] = 1 # Normalize the beam to unity
options['pybeams_fits.sky_rotation'] = 1
if timeint is None:
timeint = DEB_TIME_INTERVAL
if freqint is None:
freqint = DEB_FREQ_INTERVAL
if smooth is None:
smooth = DE_SMOOTHING
if smooth:
kw['stefcal_diffgain.freqint'] = kw['stefcal_diffgain.timeint'] = 5
kw['stefcal_diffgain.freqsmooth'] = freqint
kw['stefcal_diffgain.timesmooth'] = timeint
else:
kw['stefcal_diffgain.freqsmooth'] = kw[
'stefcal_diffgain.timesmooth'] = 3
kw['stefcal_diffgain.freqint'] = freqint
kw['stefcal_diffgain.timeint'] = timeint
stefcal.stefcal(section=tdlsec,
diffgains=False,
output="CORR_RES",
restore=dict(niter=3000),
restore_lsm=True,
apply_only=False,
flag_threshold=(1, 0.5),
options=options,
args=args,
**kw)
def cal_ms ():
"""cal_ms: runs a calibration loop over the current MS"""
# initialize the calibration "step" counter and "label". These are used
# to auto-generate output filenames; cal.stefcal() below automatically increments
# cal.STEP
v.STEP = 0
# set the superglobal LSM variable. See explanation for "v." in the text
v.LSM = LSM0
# info(), warn() and abort() are Pyxis functions for writing output to the log
info("########## solving for G with initial LSM")
# do one stefcal run with the initial model, produce corrected residuals,
# do simple flagging on the residuals. See cal.stefcal() for full docs.
stefcal.stefcal(stefcal_reset_all=True,output="CORR_RES",restore=False,flag_threshold=(1,.5))
# If we got to this point, then assume things are rolling along fine.
# Copy the recipe and config to the destination directory for future reference
# (very useful when modifying recipes, so you can keep track of what settings
# are responsible for what output!)
xo.sh("cp pyxis*.py pyxis*.conf ${mqt.TDLCONFIG} $DESTDIR",shell=True)
info("########## remaking data image, running source finder and updating model")
# apply previous stefcal solutions to produce a corrected data image.
# Make a restored image with 2000 CLEAN iterations.
stefcal.stefcal(apply_only=True,output="CORR_DATA",plotvis=False,
restore=dict(niter=2000),restore_lsm=False)
## now run pybdsm on the restored image, write output to a new LSM file
lsm.pybdsm_search(threshold=5,output=LSM1)
# the new sky model becomes the "current" LSM
v.LSM = LSM1
# note that in principle this step is not needed (may as well go on directly
# to the dE solutions, below), but it is instructive, as it will produce an intermediate-stage
# image. But to save time, set restore=False to skip making clean images
info("########## recalibrating with new model")
stefcal.stefcal(stefcal_reset_all=True,output="CORR_RES",restore=False,flag_threshold=(1,.5));
# if we have a "reference" sky model configured, use it to set dE tags on
# sources in our new sky model. Sources with dE tags will have DDE solutions.
# The cal.transfer_tags() function sets the specified tag on any source
# near enough to a reference model source with that tag.
if LSMREF:
lsm.transfer_tags(LSMREF,LSM,tags="dE",tolerance=45*ARCSEC)
info("########## recalibrating with dEs")
# re-run stefcal with the new model (and with direction dependent
# solutions on dE-tagged sources, if any). Make a corrected residual
# image, run CLEAN on it, restore the LSM sources back into the
# image
stefcal.stefcal(stefcal_reset_all=True,diffgains=True,
output="CORR_RES",restore=dict(niter=1000),
restore_lsm=True,flag_threshold=(1,.5))
# no reference LSM? then still need to make a restored image from the step above
else:
imager.make_image(dirty=False,restore=dict(niter=1000),restore_lsm=True);
# the resulting image filename is in cal.FULLREST_IMAGE,
# add it to our image list file
IMAGE_LIST.add("${imager.FULLREST_IMAGE}");
def c_cal(goto_step=1):
"""Calibration for C-config data"""
## initial calibration
if goto_step > 1:
info("########## restarting calibration from step $goto_step")
# Calibration step -- this is just a user-defined label used in filenames (etc. "blahblah_s1"), which serves to keep the output from each step
# of a pipeline separate. If this is numeric, then functions such as stefcal.stefcal() will increment it automatically first thing. Otherwise you can set
# it yourself to some more fancy label. Here we also provide a way to hop to particular step via goto_step
v.STEP = goto_step - 1
if goto_step < 2:
# set the superglobal LSM
v.LSM = LSM0
info("########## solving for G with initial LSM")
# no w-proj for dirty map to save time
stefcal.stefcal(stefcal_reset_all=True,
dirty=dict(wprojplanes=0, npix=NPIX),
restore=dict(npix=NPIX,
threshold=CLEAN_THRESH[0],
wprojplanes=128))
info("########## running source finder and updating model")
## now run pybdsm on restored image, output LSM will be given by variable cal.PYBDSM_OUTPUT
### NB: select on radius to exclude any artefacts picked up around 3C147 itself
lsm.pybdsm_search(thresh_pix=THRESH_PIX,
thresh_isl=THRESH_ISL,
select="r.gt.30s")
### merge new sources into sky model, give it a new name ($LSM1)
lsm.tigger_convert("$LSM -a ${lsm.PYBDSM_OUTPUT} $LSM1 --rename -f")
if goto_step < 3:
info("########## solving for G+dE with updated LSM (initial+pybdsm)")
v.LSM = LSM1
# now, set dE tags on sources
lsm.transfer_tags(LSMREF, LSM, tags="dE", tolerance=45 * ARCSEC)
stefcal.stefcal(stefcal_reset_all=True,
diffgains=True,
dirty=dict(wprojplanes=0, npix=NPIX))
if goto_step < 4:
info("########## re-solving for G to apply IFR solutions")
v.LSM = LSM1
stefcal.stefcal(diffgains=True,
diffgain_apply_only=True,
dirty=dict(wprojplanes=0, npix=NPIX),
restore=dict(npix=NPIX, threshold=CLEAN_THRESH[1]))
info("########## adding clean components to LSM")
ff = pyfits.open(imager.MODEL_IMAGE)
dd = ff[0].data
dd *= 1.0769 # scale up to compensate for selfcal flux suppression
# dd[dd<0] = 0; # remove negative components
ff.writeto(LSM_CCMODEL, clobber=True)
# add model image to LSM
lsm.tigger_convert("$LSM $LSM2 --add-brick=ccmodel:$LSM_CCMODEL:2 -f")
if goto_step < 5:
info("########## re-running DD solutions")
v.LSM = LSM2
# make final image
stefcal.stefcal(dirty=dict(wprojplanes=0, npix=NPIX),
diffgains=True,
restore=dict(npix=NPIX, threshold=CLEAN_THRESH[2]),
label="dE")
# make per-channel cube
makecube(NPIX)
# make noise images
makenoise()
def jointcal_g (): stefcal.stefcal(reset=True,dirty=dict(wprojplanes=0,npix=NPIX),restore=False);
def jointcal_de_final ():
stefcal.stefcal(gain_reset=ALWAYS_RESET,diffgain_reset=ALWAYS_RESET,
diffgains=True,dirty=dict(wprojplanes=0,npix=NPIX),restore=False);
def jointcal_g():
stefcal.stefcal(reset=True,
dirty=dict(wprojplanes=0, npix=NPIX),
restore=False)
def jointcal_de_reset():
stefcal.stefcal(gain_reset=True,
diffgain_reset=True,
diffgains=True,
dirty=dict(wprojplanes=0, npix=NPIX),
restore=False)
def jointcal_de_final():
stefcal.stefcal(gain_reset=ALWAYS_RESET,
diffgain_reset=ALWAYS_RESET,
diffgains=True,
dirty=dict(wprojplanes=0, npix=NPIX),
restore=False)
def selfcal_loop_pipeline(goto_step=0.):
if not os.path.isdir(v.DESTDIR):
os.makedirs(v.DESTDIR)
#copy data to corrected data column to run clean
if goto_step <= 0.:
v.STEP=1
info("########## adding columns %i"%v.STEP)
for msFile in sorted(v.MS_List):pyrap.tables.addImagingColumns(msFile)
if goto_step <= 0.5:
v.STEP=1
info("########## copying data to corrected data column step %i"%v.STEP)
pper("MS",copy_data_to_corrected_data)
#run a basic clean to perform source finding on
if goto_step <= 1:
v.STEP=1
info("########## making initial clean image step %i"%v.STEP)
wscleanDict={
'name':'wsclean',
'datacolumn':'CORRECTED_DATA',
'size': '1024 1024',
#'scale': 330./3600.,
'scale': 0.11,
'niter': 250,
'threshold': 1.,
'pol': 'I',
#'weight': 'natural',
'weight': 'briggs 0.0',
'nwlayers':32,
'channelrange': '%i %i'%(startChan,endChan),
'channelsout': 1}
for msFile in sorted(v.MS_List):
msBase=msFile.split('/')[-1].split('.MS')[0]
wscleanDict['name']=msBase+'_s%i'%v.STEP
run_wsclean(msFile,wscleanDict)
fitsfiles=glob.glob('*.fits')
for ff in fitsfiles: shutil.move(ff,v.DESTDIR)
#derive an initial sky model from the clean'd image
if goto_step <= 1.5:
v.STEP=1
fitsfiles=glob.glob(v.DESTDIR+'/*_s1-image.fits')
for ff in sorted(fitsfiles):
olsm=ff.split('-image.fits')[0]+'.lsm'
lsm.pybdsm_search(image=ff,output=olsm,thresh_pix=20,thresh_isl=15)
lsm.tigger_convert(olsm)
#run stefcal using the initial sky model
if goto_step <= 2:
v.STEP=2
lsm0files=glob.glob(v.DESTDIR+'/*_s1.lsm.html')
wscleanDict={
'name':'wsclean',
'datacolumn':'CORRECTED_DATA',
'size': '1024 1024',
#'scale': 330./3600.,
'scale': 0.11,
'niter': 500,
'threshold': 1.,
'pol': 'I',
#'weight': 'natural',
'weight': 'briggs 0.0',
'nwlayers':32,
'channelrange': '%i %i'%(startChan,endChan),
'channelsout': 1}
for msFile in sorted(v.MS_List):
msBase=msFile.split('/')[-1].split('.MS')[0]
lsm0=filter(lambda x: x.split('/')[-1].startswith(msBase),lsm0files)[0]
v.LSM=lsm0
v.MS=msFile
stefcal.STEFCAL_STEP_INCR = 0 # we set the step counter explicitly
info("########## initial calibration of %s using sky model %s step %i"%(msFile,lsm0,v.STEP))
stefcal.stefcal(output='CORR_DATA',dirty=False,restore=False,section='calico-stefcal')
info("########## making clean image step %i"%v.STEP)
wscleanDict['name']=msBase+'_s%i'%v.STEP
run_wsclean(msFile,wscleanDict)
fitsfiles=glob.glob('*.fits')
for ff in fitsfiles: shutil.move(ff,v.DESTDIR)
#derive an second sky model from the clean'd image
if goto_step <= 2.5:
v.STEP=2
fitsfiles=glob.glob(v.DESTDIR+'/*_s2-image.fits')
for ff in sorted(fitsfiles):
olsm=ff.split('-image.fits')[0]+'.lsm'
lsm.pybdsm_search(image=ff,output=olsm,thresh_pix=15,thresh_isl=10)
lsm.tigger_convert(olsm)
#run stefcal using the second sky model
if goto_step <= 3:
v.STEP=3
lsm1files=glob.glob(v.DESTDIR+'/*_s2.lsm.html')
wscleanDict={
'name':'wsclean',
'datacolumn':'CORRECTED_DATA',
'size': '1024 1024',
#'scale': 330./3600.,
'scale': 0.11,
'niter': 1000,
'threshold': 1.,
'pol': 'I',
#'weight': 'natural',
'weight': 'briggs 0.0',
'nwlayers':32,
'channelrange': '%i %i'%(startChan,endChan),
'channelsout': 1}
for msFile in sorted(v.MS_List):
x.sh('export LC_NUMERIC=en_GB.utf8')
msBase=msFile.split('/')[-1].split('.MS')[0]
lsm1=filter(lambda x: x.split('/')[-1].startswith(msBase),lsm1files)[0]
v.LSM=lsm1
v.MS=msFile
stefcal.STEFCAL_STEP_INCR = 0 # we set the step counter explicitly
info("########## second calibration of %s using sky model %s step %i"%(msFile,lsm1,v.STEP))
stefcal.stefcal(output='CORR_DATA',dirty=False,restore=False,section='calico-stefcal')
info("########## making clean image step %i"%v.STEP)
wscleanDict['name']=msBase+'_s%i'%v.STEP
run_wsclean(msFile,wscleanDict)
fitsfiles=glob.glob('*.fits')
for ff in fitsfiles: shutil.move(ff,v.DESTDIR)
#derive an sky model from the clean'd image
if goto_step <= 3.5:
v.STEP=3
fitsfiles=glob.glob(v.DESTDIR+'/*_s3-image.fits')
for ff in sorted(fitsfiles):
olsm=ff.split('-image.fits')[0]+'.lsm'
lsm.pybdsm_search(image=ff,output=olsm,thresh_pix=12,thresh_isl=5)
lsm.tigger_convert(olsm)
#run stefcal using the third sky model and produce final complex, multi-frequency images
if goto_step <= 4:
v.STEP=4
lsm2files=glob.glob(v.DESTDIR+'/*_s3.lsm.html')
wscleanDict={
'name':'wsclean',
'datacolumn':'CORRECTED_DATA',
'size': '1024 1024',
#'scale': 330./3600.,
'scale': 0.11,
'niter': 1000,
'threshold': 1.,
'pol': 'xx,xy,yx,yy',
#'weight': 'natural',
'weight': 'briggs 0.0',
'nwlayers':32,
'channelrange': '%i %i'%(startChan,endChan),
'channelsout': nsubbands,
'joinpolarizations': None}
for msFile in sorted(v.MS_List):
x.sh('export LC_NUMERIC=en_GB.utf8')
msBase=msFile.split('/')[-1].split('.MS')[0]
lsm2=filter(lambda x: x.split('/')[-1].startswith(msBase),lsm2files)[0]
v.LSM=lsm2
v.MS=msFile
stefcal.STEFCAL_STEP_INCR = 0 # we set the step counter explicitly
info("########## second calibration of %s using sky model %s step %i"%(msFile,lsm2,v.STEP))
stefcal.stefcal(output='CORR_DATA',dirty=False,restore=False,section='calico-stefcal')
wscleanDict['name']=msBase+'_s%i'%v.STEP
run_wsclean(msFile,wscleanDict)
fitsfiles=glob.glob('*.fits')
for ff in fitsfiles: shutil.move(ff,v.DESTDIR)
#clean up
if goto_step <= 5:
gaulfiles=glob.glob('*.gaul')
for gf in gaulfiles: shutil.move(gf,v.DESTDIR)
def phase_precal_selfcal_pipeline(goto_step=3.):
useStep=1
skipCalMSlist=[ 'zen.2455819.25927.uvcRREM',
'zen.2455819.27319.uvcRREM',
'zen.2455819.32190.uvcRREM',
'zen.2455819.34278.uvcRREM',
'zen.2455819.34974.uvcRREM',
'zen.2455819.38454.uvcRREM',
'zen.2455819.40542.uvcRREM',
'zen.2455819.41238.uvcRREM',
'zen.2455819.41934.uvcRREM',
'zen.2455819.42630.uvcRREM',
'zen.2455819.73947.uvcRREM',
'zen.2455819.74643.uvcRREM',
'zen.2455819.75339.uvcRREM',
'zen.2455819.76035.uvcRREM',
'zen.2455819.78819.uvcRREM',
'zen.2455819.79515.uvcRREM',
'zen.2455819.80211.uvcRREM',
'zen.2455819.80906.uvcRREM'] #MS files to skip calibration on
skipCleanMSlist=['zen.2455819.79515.uvcRREM'] #problematic MS files when imaging multi-freq
if not os.path.isdir(v.DESTDIR):
os.makedirs(v.DESTDIR)
if goto_step <= 0.:
v.STEP=1
info("########## adding columns %i"%v.STEP)
for msFile in sorted(v.MS_List):pyrap.tables.addImagingColumns(msFile)
#copy data to corrected data column to run clean
if goto_step <= 0.5:
v.STEP=1
info("########## copying data to corrected data column step %i"%v.STEP)
pper("MS",copy_data_to_corrected_data)
#run a basic clean to perform source finding on
if goto_step <= 1:
v.STEP=1
info("########## making initial clean image step %i"%v.STEP)
wscleanDict={
'name':'wsclean',
'datacolumn':'CORRECTED_DATA',
'size': '1024 1024',
#'scale': 330./3600.,
'scale': 0.11,
'niter': 250,
'threshold': 100.,
'pol': 'I',
#'weight': 'natural',
'weight': 'briggs 0.0',
'nwlayers':32,
'channelrange': '%i %i'%(startChan,endChan),
'channelsout': 1}
for msFile in sorted(v.MS_List):
msBase=msFile.split('/')[-1].split('.MS')[0]
wscleanDict['name']=msBase+'_s%i'%v.STEP
run_wsclean(msFile,wscleanDict)
fitsfiles=glob.glob('*.fits')
for ff in fitsfiles: shutil.move(ff,v.DESTDIR)
#derive a shallow depth sky model from the clean'd image
if goto_step <= 1.5:
v.STEP=1
fitsfiles=glob.glob(v.DESTDIR+'/*_s1-image.fits')
for ff in sorted(fitsfiles):
olsm=ff.split('-image.fits')[0]+'.lsm'
lsm.pybdsm_search(image=ff,output=olsm,thresh_pix=20,thresh_isl=15)
lsm.tigger_convert(olsm)
#run stefcal using the sky model, produce a Stokes I image for source finding
if goto_step <= 2:
v.STEP=2
lsm0files=glob.glob(v.DESTDIR+'/*_s1.lsm.html')
wscleanDict={
'name':'wsclean',
'datacolumn':'CORRECTED_DATA',
'size': '1024 1024',
#'scale': 330./3600.,
'scale': 0.11,
'niter': 250,
'threshold': 100.,
'pol': 'I',
#'weight': 'natural',
'weight': 'briggs 0.0',
'nwlayers':32,
'channelrange': '%i %i'%(startChan,endChan),
'channelsout': 1}
for msFile in sorted(v.MS_List):
msBase=msFile.split('/')[-1].split('.MS')[0]
if msBase in skipCalMSlist:
info('Skipping this MS file')
else:
x.sh('export LC_NUMERIC=en_GB.utf8')
lsm0=filter(lambda x: x.split('/')[-1].startswith(msBase),lsm0files)[0]
v.LSM=lsm0
v.MS=msFile
stefcal.STEFCAL_STEP_INCR = 0 # we set the step counter explicitly
info("########## initial calibration of %s using sky model %s step %i"%(msFile,lsm0,v.STEP))
stefcal.stefcal(output='CORR_DATA',dirty=False,restore=False,section='calico-stefcal')
info("########## making clean image step %i"%v.STEP)
wscleanDict['name']=msBase+'_s%i'%v.STEP
run_wsclean(msFile,wscleanDict)
fitsfiles=glob.glob('*.fits')
for ff in fitsfiles: shutil.move(ff,v.DESTDIR)
#derive a deeper depth sky model from the clean'd image
if goto_step <= 2.5:
v.STEP=2
fitsfiles=glob.glob(v.DESTDIR+'/*_s2-image.fits')
for ff in sorted(fitsfiles):
olsm=ff.split('-image.fits')[0]+'.lsm'
lsm.pybdsm_search(image=ff,output=olsm,thresh_pix=10,thresh_isl=7)
lsm.tigger_convert(olsm)
#generate multi-frequency, complex images for image domain corrections
if goto_step <= 3:
v.STEP=3
wscleanDict={
'name':'wsclean',
'datacolumn':'CORRECTED_DATA',
'size': '1024 1024',
#'scale': 330./3600.,
'scale': 0.11,
#'niter': 1000,
'niter': 1,
'threshold': 100.,
'pol': 'xx,xy,yx,yy',
'weight': 'natural',
#'weight': 'briggs 0.0',
'nwlayers':32,
'channelrange': '%i %i'%(startChan,endChan),
'channelsout': nsubbands,
'joinpolarizations': None}
for msFile in sorted(v.MS_List):
x.sh('export LC_NUMERIC=en_GB.utf8')
msBase=msFile.split('/')[-1].split('.MS')[0]
if msBase in skipCleanMSlist: wscleanDict['niter']=1
else: wscleanDict['niter']=1
#else: wscleanDict['niter']=1000
info("########## making clean image step %i"%v.STEP)
wscleanDict['name']=msBase+'_s%i'%v.STEP
run_wsclean(msFile,wscleanDict)
fitsfiles=glob.glob('*.fits')
for ff in fitsfiles: shutil.move(ff,v.DESTDIR)
def selfcal_loop_pipeline(goto_step=0.):
if not os.path.isdir(v.DESTDIR):
os.makedirs(v.DESTDIR)
#copy data to corrected data column to run clean
if goto_step <= 0.:
v.STEP = 1
info("########## adding columns %i" % v.STEP)
for msFile in sorted(v.MS_List):
pyrap.tables.addImagingColumns(msFile)
if goto_step <= 0.5:
v.STEP = 1
info("########## copying data to corrected data column step %i" %
v.STEP)
pper("MS", copy_data_to_corrected_data)
#run a basic clean to perform source finding on
if goto_step <= 1:
v.STEP = 1
info("########## making initial clean image step %i" % v.STEP)
wscleanDict = {
'name': 'wsclean',
'datacolumn': 'CORRECTED_DATA',
'size': '1024 1024',
#'scale': 330./3600.,
'scale': 0.11,
'niter': 250,
'threshold': 1.,
'pol': 'I',
#'weight': 'natural',
'weight': 'briggs 0.0',
'nwlayers': 32,
'channelrange': '%i %i' % (startChan, endChan),
'channelsout': 1
}
for msFile in sorted(v.MS_List):
msBase = msFile.split('/')[-1].split('.MS')[0]
wscleanDict['name'] = msBase + '_s%i' % v.STEP
run_wsclean(msFile, wscleanDict)
fitsfiles = glob.glob('*.fits')
for ff in fitsfiles:
shutil.move(ff, v.DESTDIR)
#derive an initial sky model from the clean'd image
if goto_step <= 1.5:
v.STEP = 1
fitsfiles = glob.glob(v.DESTDIR + '/*_s1-image.fits')
for ff in sorted(fitsfiles):
olsm = ff.split('-image.fits')[0] + '.lsm'
lsm.pybdsm_search(image=ff,
output=olsm,
thresh_pix=20,
thresh_isl=15)
lsm.tigger_convert(olsm)
#run stefcal using the initial sky model
if goto_step <= 2:
v.STEP = 2
lsm0files = glob.glob(v.DESTDIR + '/*_s1.lsm.html')
wscleanDict = {
'name': 'wsclean',
'datacolumn': 'CORRECTED_DATA',
'size': '1024 1024',
#'scale': 330./3600.,
'scale': 0.11,
'niter': 500,
'threshold': 1.,
'pol': 'I',
#'weight': 'natural',
'weight': 'briggs 0.0',
'nwlayers': 32,
'channelrange': '%i %i' % (startChan, endChan),
'channelsout': 1
}
for msFile in sorted(v.MS_List):
msBase = msFile.split('/')[-1].split('.MS')[0]
lsm0 = filter(lambda x: x.split('/')[-1].startswith(msBase),
lsm0files)[0]
v.LSM = lsm0
v.MS = msFile
stefcal.STEFCAL_STEP_INCR = 0 # we set the step counter explicitly
info(
"########## initial calibration of %s using sky model %s step %i"
% (msFile, lsm0, v.STEP))
stefcal.stefcal(output='CORR_DATA',
dirty=False,
restore=False,
section='calico-stefcal')
info("########## making clean image step %i" % v.STEP)
wscleanDict['name'] = msBase + '_s%i' % v.STEP
run_wsclean(msFile, wscleanDict)
fitsfiles = glob.glob('*.fits')
for ff in fitsfiles:
shutil.move(ff, v.DESTDIR)
#derive an second sky model from the clean'd image
if goto_step <= 2.5:
v.STEP = 2
fitsfiles = glob.glob(v.DESTDIR + '/*_s2-image.fits')
for ff in sorted(fitsfiles):
olsm = ff.split('-image.fits')[0] + '.lsm'
lsm.pybdsm_search(image=ff,
output=olsm,
thresh_pix=15,
thresh_isl=10)
lsm.tigger_convert(olsm)
#run stefcal using the second sky model
if goto_step <= 3:
v.STEP = 3
lsm1files = glob.glob(v.DESTDIR + '/*_s2.lsm.html')
wscleanDict = {
'name': 'wsclean',
'datacolumn': 'CORRECTED_DATA',
'size': '1024 1024',
#'scale': 330./3600.,
'scale': 0.11,
'niter': 1000,
'threshold': 1.,
'pol': 'I',
#'weight': 'natural',
'weight': 'briggs 0.0',
'nwlayers': 32,
'channelrange': '%i %i' % (startChan, endChan),
'channelsout': 1
}
for msFile in sorted(v.MS_List):
x.sh('export LC_NUMERIC=en_GB.utf8')
msBase = msFile.split('/')[-1].split('.MS')[0]
lsm1 = filter(lambda x: x.split('/')[-1].startswith(msBase),
lsm1files)[0]
v.LSM = lsm1
v.MS = msFile
stefcal.STEFCAL_STEP_INCR = 0 # we set the step counter explicitly
info(
"########## second calibration of %s using sky model %s step %i"
% (msFile, lsm1, v.STEP))
stefcal.stefcal(output='CORR_DATA',
dirty=False,
restore=False,
section='calico-stefcal')
info("########## making clean image step %i" % v.STEP)
wscleanDict['name'] = msBase + '_s%i' % v.STEP
run_wsclean(msFile, wscleanDict)
fitsfiles = glob.glob('*.fits')
for ff in fitsfiles:
shutil.move(ff, v.DESTDIR)
#derive an sky model from the clean'd image
if goto_step <= 3.5:
v.STEP = 3
fitsfiles = glob.glob(v.DESTDIR + '/*_s3-image.fits')
for ff in sorted(fitsfiles):
olsm = ff.split('-image.fits')[0] + '.lsm'
lsm.pybdsm_search(image=ff,
output=olsm,
thresh_pix=12,
thresh_isl=5)
lsm.tigger_convert(olsm)
#run stefcal using the third sky model and produce final complex, multi-frequency images
if goto_step <= 4:
v.STEP = 4
lsm2files = glob.glob(v.DESTDIR + '/*_s3.lsm.html')
wscleanDict = {
'name': 'wsclean',
'datacolumn': 'CORRECTED_DATA',
'size': '1024 1024',
#'scale': 330./3600.,
'scale': 0.11,
'niter': 1000,
'threshold': 1.,
'pol': 'xx,xy,yx,yy',
#'weight': 'natural',
'weight': 'briggs 0.0',
'nwlayers': 32,
'channelrange': '%i %i' % (startChan, endChan),
'channelsout': nsubbands,
'joinpolarizations': None
}
for msFile in sorted(v.MS_List):
x.sh('export LC_NUMERIC=en_GB.utf8')
msBase = msFile.split('/')[-1].split('.MS')[0]
lsm2 = filter(lambda x: x.split('/')[-1].startswith(msBase),
lsm2files)[0]
v.LSM = lsm2
v.MS = msFile
stefcal.STEFCAL_STEP_INCR = 0 # we set the step counter explicitly
info(
"########## second calibration of %s using sky model %s step %i"
% (msFile, lsm2, v.STEP))
stefcal.stefcal(output='CORR_DATA',
dirty=False,
restore=False,
section='calico-stefcal')
wscleanDict['name'] = msBase + '_s%i' % v.STEP
run_wsclean(msFile, wscleanDict)
fitsfiles = glob.glob('*.fits')
for ff in fitsfiles:
shutil.move(ff, v.DESTDIR)
#clean up
if goto_step <= 5:
gaulfiles = glob.glob('*.gaul')
for gf in gaulfiles:
shutil.move(gf, v.DESTDIR)
def phase_precal_selfcal_pipeline(goto_step=3.):
useStep = 1
skipCalMSlist = [
'zen.2455819.25927.uvcRREM', 'zen.2455819.27319.uvcRREM',
'zen.2455819.32190.uvcRREM', 'zen.2455819.34278.uvcRREM',
'zen.2455819.34974.uvcRREM', 'zen.2455819.38454.uvcRREM',
'zen.2455819.40542.uvcRREM', 'zen.2455819.41238.uvcRREM',
'zen.2455819.41934.uvcRREM', 'zen.2455819.42630.uvcRREM',
'zen.2455819.73947.uvcRREM', 'zen.2455819.74643.uvcRREM',
'zen.2455819.75339.uvcRREM', 'zen.2455819.76035.uvcRREM',
'zen.2455819.78819.uvcRREM', 'zen.2455819.79515.uvcRREM',
'zen.2455819.80211.uvcRREM', 'zen.2455819.80906.uvcRREM'
] #MS files to skip calibration on
skipCleanMSlist = ['zen.2455819.79515.uvcRREM'
] #problematic MS files when imaging multi-freq
if not os.path.isdir(v.DESTDIR):
os.makedirs(v.DESTDIR)
if goto_step <= 0.:
v.STEP = 1
info("########## adding columns %i" % v.STEP)
for msFile in sorted(v.MS_List):
pyrap.tables.addImagingColumns(msFile)
#copy data to corrected data column to run clean
if goto_step <= 0.5:
v.STEP = 1
info("########## copying data to corrected data column step %i" %
v.STEP)
pper("MS", copy_data_to_corrected_data)
#run a basic clean to perform source finding on
if goto_step <= 1:
v.STEP = 1
info("########## making initial clean image step %i" % v.STEP)
wscleanDict = {
'name': 'wsclean',
'datacolumn': 'CORRECTED_DATA',
'size': '1024 1024',
#'scale': 330./3600.,
'scale': 0.11,
'niter': 250,
'threshold': 100.,
'pol': 'I',
#'weight': 'natural',
'weight': 'briggs 0.0',
'nwlayers': 32,
'channelrange': '%i %i' % (startChan, endChan),
'channelsout': 1
}
for msFile in sorted(v.MS_List):
msBase = msFile.split('/')[-1].split('.MS')[0]
wscleanDict['name'] = msBase + '_s%i' % v.STEP
run_wsclean(msFile, wscleanDict)
fitsfiles = glob.glob('*.fits')
for ff in fitsfiles:
shutil.move(ff, v.DESTDIR)
#derive a shallow depth sky model from the clean'd image
if goto_step <= 1.5:
v.STEP = 1
fitsfiles = glob.glob(v.DESTDIR + '/*_s1-image.fits')
for ff in sorted(fitsfiles):
olsm = ff.split('-image.fits')[0] + '.lsm'
lsm.pybdsm_search(image=ff,
output=olsm,
thresh_pix=20,
thresh_isl=15)
lsm.tigger_convert(olsm)
#run stefcal using the sky model, produce a Stokes I image for source finding
if goto_step <= 2:
v.STEP = 2
lsm0files = glob.glob(v.DESTDIR + '/*_s1.lsm.html')
wscleanDict = {
'name': 'wsclean',
'datacolumn': 'CORRECTED_DATA',
'size': '1024 1024',
#'scale': 330./3600.,
'scale': 0.11,
'niter': 250,
'threshold': 100.,
'pol': 'I',
#'weight': 'natural',
'weight': 'briggs 0.0',
'nwlayers': 32,
'channelrange': '%i %i' % (startChan, endChan),
'channelsout': 1
}
for msFile in sorted(v.MS_List):
msBase = msFile.split('/')[-1].split('.MS')[0]
if msBase in skipCalMSlist:
info('Skipping this MS file')
else:
x.sh('export LC_NUMERIC=en_GB.utf8')
lsm0 = filter(lambda x: x.split('/')[-1].startswith(msBase),
lsm0files)[0]
v.LSM = lsm0
v.MS = msFile
stefcal.STEFCAL_STEP_INCR = 0 # we set the step counter explicitly
info(
"########## initial calibration of %s using sky model %s step %i"
% (msFile, lsm0, v.STEP))
stefcal.stefcal(output='CORR_DATA',
dirty=False,
restore=False,
section='calico-stefcal')
info("########## making clean image step %i" % v.STEP)
wscleanDict['name'] = msBase + '_s%i' % v.STEP
run_wsclean(msFile, wscleanDict)
fitsfiles = glob.glob('*.fits')
for ff in fitsfiles:
shutil.move(ff, v.DESTDIR)
#derive a deeper depth sky model from the clean'd image
if goto_step <= 2.5:
v.STEP = 2
fitsfiles = glob.glob(v.DESTDIR + '/*_s2-image.fits')
for ff in sorted(fitsfiles):
olsm = ff.split('-image.fits')[0] + '.lsm'
lsm.pybdsm_search(image=ff,
output=olsm,
thresh_pix=10,
thresh_isl=7)
lsm.tigger_convert(olsm)
#generate multi-frequency, complex images for image domain corrections
if goto_step <= 3:
v.STEP = 3
wscleanDict = {
'name': 'wsclean',
'datacolumn': 'CORRECTED_DATA',
'size': '1024 1024',
#'scale': 330./3600.,
'scale': 0.11,
#'niter': 1000,
'niter': 1,
'threshold': 100.,
'pol': 'xx,xy,yx,yy',
'weight': 'natural',
#'weight': 'briggs 0.0',
'nwlayers': 32,
'channelrange': '%i %i' % (startChan, endChan),
'channelsout': nsubbands,
'joinpolarizations': None
}
for msFile in sorted(v.MS_List):
x.sh('export LC_NUMERIC=en_GB.utf8')
msBase = msFile.split('/')[-1].split('.MS')[0]
if msBase in skipCleanMSlist: wscleanDict['niter'] = 1
else: wscleanDict['niter'] = 1
#else: wscleanDict['niter']=1000
info("########## making clean image step %i" % v.STEP)
wscleanDict['name'] = msBase + '_s%i' % v.STEP
run_wsclean(msFile, wscleanDict)
fitsfiles = glob.glob('*.fits')
for ff in fitsfiles:
shutil.move(ff, v.DESTDIR)
#flaglist[0:start_freq_channel] = False
#flaglist[end_freq_channel:] = False
antA = [h5.inputs.index(inpA) for inpA, inpB in h5.corr_products]
antB = [h5.inputs.index(inpB) for inpA, inpB in h5.corr_products]
N_ants = len(h5.ants)
#full_vis = np.concatenate((vis, vis.conj()), axis=-1)
full_antA = np.r_[antA, antB]
full_antB = np.r_[antB, antA]
weights= np.abs(1./np.angle(absstd(vis[:,:,:],axis=0)))
weights= np.concatenate((weights, weights), axis=-1)
# use vector mean == np.mean on visabilitys
# but use angle mean on solutions/phase change.
gains = stefcal.stefcal( np.concatenate( (np.mean(vis,axis=0), np.mean(vis.conj(),axis=0)), axis=-1) , N_ants, full_antA, full_antB, num_iters=50,weights=weights)
calfac = 1./(gains[np.newaxis][:,:,full_antA]*gains[np.newaxis][:,:,full_antB].conj())
h5.select(channels=slice(start_freq_channel,end_freq_channel),pol=pol,corrprods='cross',scans='track')
data = np.zeros((h5.shape[0:3:2]),dtype=np.complex)
i = 0
for scan in h5.scans():
print scan
if np.all(h5.sensor['DBE/auto-delay'] == '0') :
print "stopping fringes for size ",h5.shape
vis = fringe_stopping(h5)
else:
vis = h5.vis[:,:,:]
data[i:i+h5.shape[0]] = mean((vis*calfac[:,:,:h5.shape[-1]])[:,flaglist,:],axis=1)
i += h5.shape[0]
figlist = []
def cal_ms():
"""cal_ms: runs a calibration loop over the current MS"""
# initialize the calibration "step" counter and "label". These are used
# to auto-generate output filenames; cal.stefcal() below automatically increments
# cal.STEP
v.STEP = 0
# set the superglobal LSM variable. See explanation for "v." in the text
v.LSM = LSM0
# info(), warn() and abort() are Pyxis functions for writing output to the log
info("########## solving for G with initial LSM")
# do one stefcal run with the initial model, produce corrected residuals,
# do simple flagging on the residuals. See cal.stefcal() for full docs.
stefcal.stefcal(stefcal_reset_all=True,
output="CORR_RES",
restore=False,
flag_threshold=(1, .5))
# If we got to this point, then assume things are rolling along fine.
# Copy the recipe and config to the destination directory for future reference
# (very useful when modifying recipes, so you can keep track of what settings
# are responsible for what output!)
xo.sh("cp pyxis*.py pyxis*.conf ${mqt.TDLCONFIG} $DESTDIR", shell=True)
info(
"########## remaking data image, running source finder and updating model"
)
# apply previous stefcal solutions to produce a corrected data image.
# Make a restored image with 2000 CLEAN iterations.
stefcal.stefcal(apply_only=True,
output="CORR_DATA",
plotvis=False,
restore=dict(niter=2000),
restore_lsm=False)
## now run pybdsm on the restored image, write output to a new LSM file
lsm.pybdsm_search(threshold=5, output=LSM1)
# the new sky model becomes the "current" LSM
v.LSM = LSM1
# note that in principle this step is not needed (may as well go on directly
# to the dE solutions, below), but it is instructive, as it will produce an intermediate-stage
# image. But to save time, set restore=False to skip making clean images
info("########## recalibrating with new model")
stefcal.stefcal(stefcal_reset_all=True,
output="CORR_RES",
restore=False,
flag_threshold=(1, .5))
# if we have a "reference" sky model configured, use it to set dE tags on
# sources in our new sky model. Sources with dE tags will have DDE solutions.
# The cal.transfer_tags() function sets the specified tag on any source
# near enough to a reference model source with that tag.
if LSMREF:
lsm.transfer_tags(LSMREF, LSM, tags="dE", tolerance=45 * ARCSEC)
info("########## recalibrating with dEs")
# re-run stefcal with the new model (and with direction dependent
# solutions on dE-tagged sources, if any). Make a corrected residual
# image, run CLEAN on it, restore the LSM sources back into the
# image
stefcal.stefcal(stefcal_reset_all=True,
diffgains=True,
output="CORR_RES",
restore=dict(niter=1000),
restore_lsm=True,
flag_threshold=(1, .5))
# no reference LSM? then still need to make a restored image from the step above
else:
imager.make_image(dirty=False,
restore=dict(niter=1000),
restore_lsm=True)
# the resulting image filename is in cal.FULLREST_IMAGE,
# add it to our image list file
IMAGE_LIST.add("${imager.FULLREST_IMAGE}")
