def c_cal (goto_step=1):
"""Calibration for C-config data"""
## initial calibration
if goto_step > 1:
info("########## restarting calibration from step $goto_step");
# Calibration step -- this is just a user-defined label used in filenames (etc. "blahblah_s1"), which serves to keep the output from each step
# of a pipeline separate. If this is numeric, then functions such as stefcal.stefcal() will increment it automatically first thing. Otherwise you can set
# it yourself to some more fancy label. Here we also provide a way to hop to particular step via goto_step
v.STEP = goto_step-1;
if goto_step < 2:
# set the superglobal LSM
v.LSM = LSM0
info("########## solving for G with initial LSM");
# no w-proj for dirty map to save time
stefcal.stefcal(stefcal_reset_all=True,
dirty=dict(wprojplanes=0,npix=NPIX),
restore=dict(npix=NPIX,threshold=CLEAN_THRESH[0],wprojplanes=128));
info("########## running source finder and updating model");
## now run pybdsm on restored image, output LSM will be given by variable cal.PYBDSM_OUTPUT
### NB: select on radius to exclude any artefacts picked up around 3C147 itself
lsm.pybdsm_search(thresh_pix=THRESH_PIX,thresh_isl=THRESH_ISL,select="r.gt.30s");
### merge new sources into sky model, give it a new name ($LSM1)
lsm.tigger_convert("$LSM -a ${lsm.PYBDSM_OUTPUT} $LSM1 --rename -f");
if goto_step < 3:
info("########## solving for G+dE with updated LSM (initial+pybdsm)");
v.LSM = LSM1
# now, set dE tags on sources
lsm.transfer_tags(LSMREF,LSM,tags="dE",tolerance=45*ARCSEC);
stefcal.stefcal(stefcal_reset_all=True,diffgains=True,dirty=dict(wprojplanes=0,npix=NPIX));
if goto_step < 4:
info("########## re-solving for G to apply IFR solutions");
v.LSM = LSM1
stefcal.stefcal(diffgains=True,diffgain_apply_only=True,
dirty=dict(wprojplanes=0,npix=NPIX),
restore=dict(npix=NPIX,threshold=CLEAN_THRESH[1]));
info("########## adding clean components to LSM");
ff = pyfits.open(imager.MODEL_IMAGE);
dd = ff[0].data;
dd *= 1.0769 # scale up to compensate for selfcal flux suppression
# dd[dd<0] = 0; # remove negative components
ff.writeto(LSM_CCMODEL,clobber=True);
# add model image to LSM
lsm.tigger_convert("$LSM $LSM2 --add-brick=ccmodel:$LSM_CCMODEL:2 -f");
if goto_step < 5:
info("########## re-running DD solutions");
v.LSM = LSM2
# make final image
stefcal.stefcal(dirty=dict(wprojplanes=0,npix=NPIX),diffgains=True,
restore=dict(npix=NPIX,threshold=CLEAN_THRESH[2]),
label="dE");
# make per-channel cube
makecube(NPIX);
# make noise images
makenoise();
def find_sources(image='${im.RESTORED_IMAGE}',
psf_image="${im.PSF_IMAGE}",
thresh_isl=None,
thresh_pix=None,
neg_isl=None,
neg_pix=None,
output='$PLSM',
reliability=False,
config=None,
outdir="$DESTDIR",
gaul="$GAUL",
**kw):
image, psf_image, output, gaul, outdir = interpolate_locals(
'image psf_image output gaul outdir')
thresh_pix = thresh_pix or THRESHOLDS[0]
thresh_isl = thresh_isl or THRESHOLDS[1]
neg_thresh_pix = neg_pix or NEG_THRESH[0]
neg_thresh_isl = neg_isl or NEG_THRESH[1]
if reliability:
x.sh(
"sourcery -i $image -p $psf_image -od $outdir -nisl=$neg_thresh_isl\
-npix=$neg_thresh_pix -pisl=$thresh_isl -ppix=$thresh_pix -ps=3 -ns=1\
-apsf -alv")
else:
lsm.pybdsm_search(image=image,
output=output,
thresh_pix=thresh_pix,
thresh_isl=thresh_isl,
**kw)
def cal_ms(lsm0='$LSM0', start=0, stop=4):
""" Run pipeline on a single MS"""
lsm0 = II(lsm0)
# Calibrate each MS
#run_cmd = lambda : calibrate(lsmname=lsm0)
#run_cmd = calibrate(lsmname=lsm0)
v.MS = CONCAT_MS
calibrate(lsmname=lsm0)
#pper("MS",run_cmd)
#calibrate(lsmname=lsm0)
# image combined MS
#ms.virtconcat(output=CONCAT_MS)
v.MS = CONCAT_MS
im.make_image(restore=True, psf=True, restore_lsm=False)
# run source finder
lsm.pybdsm_search(thresh_pix=5 , thresh_isl=3)
v.LSM = lsm.PYBDSM_OUTPUT
x.sh("tigger-convert --append $LSM $LSM0 $LSMFINAL -f")
# make final restored map
x.sh("tigger-restore ${im.RESTORED_IMAGE} $LSM0 ${im.FULLREST_IMAGE} -f ")
def cal_DI(msname="$MS",
lsm0='$LSM0',
timeint=None,
freqint=None,
smooth=None,
start=1,
stop=5,
**kw):
""" Run pipeline on a single MS"""
msname, lsm0 = interpolate_locals("msname lsm0")
# Use the initial model to do the direction independent calibration.
calibrate_DI(msname, lsm0)
im.make_image(restore=True, psf=True, restore_lsm=False)
lsm.pybdsm_search(thresh_pix=5, thresh_isl=3)
v.LSM = lsm.PYBDSM_OUTPUT
lsm1 = v.LSM
x.sh("tigger-convert --append $LSM $lsm0 $LSMFINAL -f")
x.sh(
"tigger-restore ${im.RESTORED_IMAGE} $LSMFINAL ${im.FULLREST_IMAGE} -f "
)
def runcal(goto_step=1):
## initial calibration
if goto_step > 1:
info("########## restarting calibration from step $goto_step")
# Calibration step -- this is just a user-defined label used in filenames (etc. "blahblah_s1"), which serves to keep the output from each step
# of a pipeline separate. If this is numeric, then functions such as stefcal.stefcal() will increment it automatically first thing. Otherwise you can set
# it yourself to some more fancy label. Here we also provide a way to hop to particular step via goto_step
v.STEP = goto_step - 1
if goto_step < 2:
# set the superglobal LSM
v.LSM = LSM0
info("########## solving for G with initial LSM")
# no w-proj for dirty map to save time
stefcal.stefcal(stefcal_reset_ifr_gains=True,
dirty=dict(wprojplanes=0),
restore=True)
info("########## running source finder and updating model")
## now run pybdsm on restored image, output LSM will be given by variable cal.PYBDSM_OUTPUT
lsm.pybdsm_search(threshold=7)
### merge new sources into sky model, give it a new name ($LSM1)
lsm.tigger_convert("$LSM -a ${lsm.PYBDSM_OUTPUT} $LSM1 --rename -f")
if goto_step < 3:
info("########## solving for G with updated LSM (initial+pybdsm)")
v.LSM = LSM1
stefcal.stefcal(dirty=dict(wprojplanes=0))
if goto_step < 4:
info("########## re-solving for G to apply IFR solutions")
stefcal.stefcal(dirty=dict(wprojplanes=0), restore=True)
info("########## adding clean components to LSM")
CCMODEL = II("ccmodel-ddid${ms.DDID}.fits")
# note the per-style variable interpolation done by the II() function
ff = pyfits.open(imager.MODEL_IMAGE)
dd = ff[0].data
dd *= 1.0769 # scale up to compensate for selfcal flux suppression
# dd[dd<0] = 0; # remove negative components
ff.writeto(CCMODEL, clobber=True)
# add model image to LSM
lsm.tigger_convert("$LSM $LSM2 --add-brick=ccmodel:$CCMODEL:2 -f")
if goto_step < 5:
info("########## solving for G with updated LSM (inital+pybdsm+cc)")
v.LSM = LSM2
stefcal.stefcal(dirty=dict(wprojplanes=0))
if goto_step < 6:
info("########## running DD solutions")
v.LSM = LSM2
# now, set dE tags on sources
lsm.transfer_tags(LSMREF, LSM, tags="dE", tolerance=45 * ARCSEC)
# make final image
stefcal.stefcal(dirty=dict(wprojplanes=0),
diffgains=True,
restore=True,
label="dE")
def runcal (goto_step=1):
## initial calibration
if goto_step > 1:
info("########## restarting calibration from step $goto_step");
# Calibration step -- this is just a user-defined label used in filenames (etc. "blahblah_s1"), which serves to keep the output from each step
# of a pipeline separate. If this is numeric, then functions such as stefcal.stefcal() will increment it automatically first thing. Otherwise you can set
# it yourself to some more fancy label. Here we also provide a way to hop to particular step via goto_step
v.STEP = goto_step-1;
if goto_step < 2:
# set the superglobal LSM
v.LSM = LSM0
info("########## solving for G with initial LSM");
# no w-proj for dirty map to save time
stefcal.stefcal(stefcal_reset_ifr_gains=True,dirty=dict(wprojplanes=0),restore=True);
info("########## running source finder and updating model");
## now run pybdsm on restored image, output LSM will be given by variable cal.PYBDSM_OUTPUT
lsm.pybdsm_search(threshold=7);
### merge new sources into sky model, give it a new name ($LSM1)
lsm.tigger_convert("$LSM -a ${lsm.PYBDSM_OUTPUT} $LSM1 --rename -f");
if goto_step < 3:
info("########## solving for G with updated LSM (initial+pybdsm)");
v.LSM = LSM1
stefcal.stefcal(dirty=dict(wprojplanes=0));
if goto_step < 4:
info("########## re-solving for G to apply IFR solutions");
stefcal.stefcal(dirty=dict(wprojplanes=0),restore=True);
info("########## adding clean components to LSM");
CCMODEL = II("ccmodel-ddid${ms.DDID}.fits"); # note the per-style variable interpolation done by the II() function
ff = pyfits.open(imager.MODEL_IMAGE);
dd = ff[0].data;
dd *= 1.0769 # scale up to compensate for selfcal flux suppression
# dd[dd<0] = 0; # remove negative components
ff.writeto(CCMODEL,clobber=True);
# add model image to LSM
lsm.tigger_convert("$LSM $LSM2 --add-brick=ccmodel:$CCMODEL:2 -f");
if goto_step < 5:
info("########## solving for G with updated LSM (inital+pybdsm+cc)");
v.LSM = LSM2
stefcal.stefcal(dirty=dict(wprojplanes=0));
if goto_step < 6:
info("########## running DD solutions");
v.LSM = LSM2
# now, set dE tags on sources
lsm.transfer_tags(LSMREF,LSM,tags="dE",tolerance=45*ARCSEC);
# make final image
stefcal.stefcal(dirty=dict(wprojplanes=0),diffgains=True,restore=True,label="dE");
def run_all():
info(
"############## Make image from the data column, Model the sky and dE tag ######################"
)
im.make_image(column="DATA", restore=True, psf=True, restore_lsm=False)
make_clean_model()
info("######################## Calibrating G+B only ####################")
cal_DI(lsm0=LSM0)
im.make_image(column="CORRECTED_DATA",
restore=True,
psf=True,
restore_lsm=False)
image = im.RESTORED_IMAGE
lsm.pybdsm_search(image, thresh_pix=5, thresh_isl=3)
v.LSM = lsm.PYBDSM_OUTPUT
lsm1 = v.LSM
make_clean_model(image=im.RESTORED_IMAGE, lsm0=LSM, threshold=3)
info(
"################# CALIBRATING DIFFENTIAL GAIN (dE): the flyswatter (O.M.Smirnov) ###################"
)
cal_DD(lsm0=LSMFINAL)
im.make_image(column="CORRECTED_DATA",
restore=True,
psf=True,
restore_lsm=False)
image = im.RESTORED_IMAGE
lsm.pybdsm_search(image, thresh_pix=5, thresh_isl=3)
v.LSM = lsm.PYBDSM_OUTPUT
lsm1 = v.LSM
make_clean_model(image=im.RESTORED_IMAGE, lsm0=LSM, threshold=3)
info(
"############## Change the apparent flux to intrinsic flux for beam calibration ######################"
)
APP_INT(lsmname=LSM0, output=LSM_INTRINSIC)
info(
"############## Do only G+B with beam and with beam ######################"
)
cal_BM(lsm0=LSM_INTRINSIC)
x.sh("tigger-restore ${im.RESTORED_IMAGE} $LSM0 ${im.FULLREST_IMAGE} -f ")
def find_sources(image='${im.RESTORED_IMAGE}',psf_image="${im.PSF_IMAGE}", thresh_isl=None,
thresh_pix=None, neg_isl=None, neg_pix=None, output='$PLSM',
reliability=False, config=None, outdir="$DESTDIR", gaul="$GAUL", **kw):
image, psf_image, output, gaul, outdir = interpolate_locals('image psf_image output gaul outdir')
thresh_pix = thresh_pix or THRESHOLDS[0]
thresh_isl = thresh_isl or THRESHOLDS[1]
neg_thresh_pix = neg_pix or NEG_THRESH[0]
neg_thresh_isl = neg_isl or NEG_THRESH[1]
if reliability:
x.sh("sourcery -i $image -p $psf_image -od $outdir -nisl=$neg_thresh_isl\
-npix=$neg_thresh_pix -pisl=$thresh_isl -ppix=$thresh_pix -ps=3 -ns=1\
-apsf -alv")
else:
lsm.pybdsm_search(image=image, output=output, thresh_pix=thresh_pix, thresh_isl=thresh_isl, **kw)
def cal_ms(lsm0='$LSM0', start=0, stop=4):
""" Run pipeline on a single MS"""
lsm0 = II(lsm0)
# Calibrate each MS
run_cmd = lambda: calibrate(lsmname=lsm0)
pper("MS", run_cmd)
# image combined MS
ms.virtconcat(output=CONCAT_MS)
v.MS = CONCAT_MS
im.make_image(restore=True, psf=True, restore_lsm=False)
# run source finder
lsm.pybdsm_search(thresh_pix=5, thresh_isl=3)
v.LSM = lsm.PYBDSM_OUTPUT
x.sh("tigger-convert --append $LSM $LSM0 $LSMFINAL -f")
# make final restored map
x.sh("tigger-restore ${im.RESTORED_IMAGE} $LSM0 ${im.FULLREST_IMAGE} -f ")
def make_clean_model(image="${im.RESTORED_IMAGE}",
psf_image="${im.PSF_IMAGE}",
lsm0="$LSM0",
threshold=7):
image, psf_image, lsm0 = interpolate_locals("image psf_image lsm0")
lsm.pybdsm_search(image, output=lsm0, threshold=threshold)
catalog = Tigger.load(lsm0)
src = catalog.sources
cs = []
for i in range(len(src)):
position = [deg(src[i].pos.ra), deg(src[i].pos.dec)]
c = correlation_factor(src[i],
psf=psf_image,
img=image,
pos_sky=position,
step=120)
cs.append(c)
for i in range(len(src)):
if cs[i] > 0.6 * max(cs):
src[i].setTag('dE', True)
catalog.save(lsm0)
def cal_ms ():
"""cal_ms: runs a calibration loop over the current MS"""
# initialize the calibration "step" counter and "label". These are used
# to auto-generate output filenames; cal.stefcal() below automatically increments
# cal.STEP
v.STEP = 0
# set the superglobal LSM variable. See explanation for "v." in the text
v.LSM = LSM0
# info(), warn() and abort() are Pyxis functions for writing output to the log
info("########## solving for G with initial LSM")
# do one stefcal run with the initial model, produce corrected residuals,
# do simple flagging on the residuals. See cal.stefcal() for full docs.
stefcal.stefcal(stefcal_reset_all=True,output="CORR_RES",restore=False,flag_threshold=(1,.5))
# If we got to this point, then assume things are rolling along fine.
# Copy the recipe and config to the destination directory for future reference
# (very useful when modifying recipes, so you can keep track of what settings
# are responsible for what output!)
xo.sh("cp pyxis*.py pyxis*.conf ${mqt.TDLCONFIG} $DESTDIR",shell=True)
info("########## remaking data image, running source finder and updating model")
# apply previous stefcal solutions to produce a corrected data image.
# Make a restored image with 2000 CLEAN iterations.
stefcal.stefcal(apply_only=True,output="CORR_DATA",plotvis=False,
restore=dict(niter=2000),restore_lsm=False)
## now run pybdsm on the restored image, write output to a new LSM file
lsm.pybdsm_search(threshold=5,output=LSM1)
# the new sky model becomes the "current" LSM
v.LSM = LSM1
# note that in principle this step is not needed (may as well go on directly
# to the dE solutions, below), but it is instructive, as it will produce an intermediate-stage
# image. But to save time, set restore=False to skip making clean images
info("########## recalibrating with new model")
stefcal.stefcal(stefcal_reset_all=True,output="CORR_RES",restore=False,flag_threshold=(1,.5));
# if we have a "reference" sky model configured, use it to set dE tags on
# sources in our new sky model. Sources with dE tags will have DDE solutions.
# The cal.transfer_tags() function sets the specified tag on any source
# near enough to a reference model source with that tag.
if LSMREF:
lsm.transfer_tags(LSMREF,LSM,tags="dE",tolerance=45*ARCSEC)
info("########## recalibrating with dEs")
# re-run stefcal with the new model (and with direction dependent
# solutions on dE-tagged sources, if any). Make a corrected residual
# image, run CLEAN on it, restore the LSM sources back into the
# image
stefcal.stefcal(stefcal_reset_all=True,diffgains=True,
output="CORR_RES",restore=dict(niter=1000),
restore_lsm=True,flag_threshold=(1,.5))
# no reference LSM? then still need to make a restored image from the step above
else:
imager.make_image(dirty=False,restore=dict(niter=1000),restore_lsm=True);
# the resulting image filename is in cal.FULLREST_IMAGE,
# add it to our image list file
IMAGE_LIST.add("${imager.FULLREST_IMAGE}");
def c_cal(goto_step=1):
"""Calibration for C-config data"""
## initial calibration
if goto_step > 1:
info("########## restarting calibration from step $goto_step")
# Calibration step -- this is just a user-defined label used in filenames (etc. "blahblah_s1"), which serves to keep the output from each step
# of a pipeline separate. If this is numeric, then functions such as stefcal.stefcal() will increment it automatically first thing. Otherwise you can set
# it yourself to some more fancy label. Here we also provide a way to hop to particular step via goto_step
v.STEP = goto_step - 1
if goto_step < 2:
# set the superglobal LSM
v.LSM = LSM0
info("########## solving for G with initial LSM")
# no w-proj for dirty map to save time
stefcal.stefcal(stefcal_reset_all=True,
dirty=dict(wprojplanes=0, npix=NPIX),
restore=dict(npix=NPIX,
threshold=CLEAN_THRESH[0],
wprojplanes=128))
info("########## running source finder and updating model")
## now run pybdsm on restored image, output LSM will be given by variable cal.PYBDSM_OUTPUT
### NB: select on radius to exclude any artefacts picked up around 3C147 itself
lsm.pybdsm_search(thresh_pix=THRESH_PIX,
thresh_isl=THRESH_ISL,
select="r.gt.30s")
### merge new sources into sky model, give it a new name ($LSM1)
lsm.tigger_convert("$LSM -a ${lsm.PYBDSM_OUTPUT} $LSM1 --rename -f")
if goto_step < 3:
info("########## solving for G+dE with updated LSM (initial+pybdsm)")
v.LSM = LSM1
# now, set dE tags on sources
lsm.transfer_tags(LSMREF, LSM, tags="dE", tolerance=45 * ARCSEC)
stefcal.stefcal(stefcal_reset_all=True,
diffgains=True,
dirty=dict(wprojplanes=0, npix=NPIX))
if goto_step < 4:
info("########## re-solving for G to apply IFR solutions")
v.LSM = LSM1
stefcal.stefcal(diffgains=True,
diffgain_apply_only=True,
dirty=dict(wprojplanes=0, npix=NPIX),
restore=dict(npix=NPIX, threshold=CLEAN_THRESH[1]))
info("########## adding clean components to LSM")
ff = pyfits.open(imager.MODEL_IMAGE)
dd = ff[0].data
dd *= 1.0769 # scale up to compensate for selfcal flux suppression
# dd[dd<0] = 0; # remove negative components
ff.writeto(LSM_CCMODEL, clobber=True)
# add model image to LSM
lsm.tigger_convert("$LSM $LSM2 --add-brick=ccmodel:$LSM_CCMODEL:2 -f")
if goto_step < 5:
info("########## re-running DD solutions")
v.LSM = LSM2
# make final image
stefcal.stefcal(dirty=dict(wprojplanes=0, npix=NPIX),
diffgains=True,
restore=dict(npix=NPIX, threshold=CLEAN_THRESH[2]),
label="dE")
# make per-channel cube
makecube(NPIX)
# make noise images
makenoise()
def jointcal (goto_step=1,last_step=10,lsmbase=None,STEPS=None):
"""Calibration for joint C and D-config data"""
info(">>>>>>>>>>>>> output directory is $OUTDIR. Please set OUTDIR explicitly to override");
# setup LSM filenames based on the full MS
# note that these get interpolated once and for all here (and the _Template definitions above
# get cancelled due to the explicit assignment here). The reason for doing it like this
# is because I don't want these names to be changing due to the templates every time the
# MS changes in a per(MS) call.
v.FULLMS = MS
LSM1 = II("$DESTDIR/$LSMBASE$SUFFIX+pybdsm.lsm.html");
LSM2 = II("$DESTDIR/$LSMBASE$SUFFIX+pybdsm2.lsm.html");
LSM3 = II("$DESTDIR/$LSMBASE$SUFFIX+pybdsm2+cc.lsm.html");
LSM_CCMODEL = II("$DESTDIR/$LSMBASE$SUFFIX+ccmodel.fits");
saveconf()
stefcal.STEFCAL_DIFFGAIN_SMOOTHING = DE_SMOOTHING if not DE_INTERVALS else None;
stefcal.STEFCAL_DIFFGAIN_INTERVALS = DE_INTERVALS;
if DE_INTERVALS and TILE:
stefcal.STEFCAL_TDLOPTS = "ms_sel.tile_size=%d"%(TILE*DE_INTERVALS[0]);
# make MS list from sub-MSs
import glob
v.MS_List = glob.glob(MS+"/SUBMSS/*MS");
info("MS list is $MS_List");
if not MS_List:
abort("No sub-MSs found");
imager.npix = NPIX = 4096
imager.cellsize = "2arcsec"
imager.wprojplanes = 128
imager.CLEAN_ALGORITHM = "csclean"
v.LSMREF = "${MS:BASE}.refmodel.lsm.html"
THRESH_PIX,THRESH_ISL = (50,10),(15,5)
CLEAN_THRESH = ".4mJy",".1mJy",".05mJy"
stefcal.STEFCAL_STEP_INCR = 0 # precvent stefcal from auto-incrementing v.STEP: we set the step counter explicitly here
if STEPS is None:
STEPS = list(numpy.arange(goto_step,last_step+.1,.5));
STEPS = map(float,STEPS);
if STEPS[0] != 1:
info("########## restarting calibration from step %.1f"%STEPS[0]);
if lsmbase:
LSMBASE = lsmbase;
## initial calibration
if 1. in STEPS:
info("########## step 1: solving for G with initial LSM");
v.LSM,v.STEP = LSM0,1
per_ms(jointcal_g);
if 1.5 in STEPS:
info("########## step 1.5: making joint image");
v.LSM,v.STEP = LSM0,1
v.MS = FULLMS
# initial model is total flux only, made from a 2x size image to catch distant sources
imager.make_image(dirty=False,stokes="I",restore=dict(npix=NPIX*2,threshold=CLEAN_THRESH[0],wprojplanes=128),restore_lsm=False);
info("########## running source finder and updating model");
## now run pybdsm on restored image, output LSM will be given by variable cal.PYBDSM_OUTPUT
### NB: select on radius to exclude any artefacts picked up around 3C147 itself
lsm.pybdsm_search(thresh_pix=THRESH_PIX[0],thresh_isl=THRESH_ISL[0],select="r.gt.30s",pol=False);
### merge new sources into sky model, give it a new name ($LSM1)
lsm.tigger_convert("$LSM -a ${lsm.PYBDSM_OUTPUT} $LSM1 --rename -f");
# if 2. in STEPS:
# info("########## step 2: repeating G solution");
# v.LSM,v.STEP = LSM1,2
# v.MS = FULLMS
# per_ms(jointcal_g);
if 2. in STEPS:
info("########## step 2: initial dE solution");
v.LSM,v.STEP = LSM1,2
v.MS = FULLMS
# now, set dE tags on sources
lsm.transfer_tags(LSMREF,LSM,tags="dE",tolerance=45*ARCSEC);
per_ms(jointcal_de_reset);
if 3. in STEPS:
info("########## step 3: re-solving for G to apply IFR solutions");
v.LSM,v.STEP = LSM1,3
v.MS = FULLMS
per_ms(jointcal_de_apply);
info("########## running source finder and updating model");
v.MS = FULLMS
imager.make_image(dirty=False,stokes="IV",restore=dict(npix=NPIX,threshold=CLEAN_THRESH[1],wprojplanes=128),restore_lsm=False);
## now run pybdsm on restored image, output LSM will be given by variable cal.PYBDSM_OUTPUT
### NB: select on radius to exclude any artefacts picked up around 3C147 itself
lsm.pybdsm_search(thresh_pix=THRESH_PIX[1],thresh_isl=THRESH_ISL[1],select="r.gt.30s");
### merge new sources into sky model, give it a new name ($LSM1)
lsm.tigger_convert("$LSM -a ${lsm.PYBDSM_OUTPUT} $LSM2 --rename -f");
if 4. in STEPS:
info("########## step 4: solving for G+dE with updated LSM (initial+pybdsm^2)");
v.MS = FULLMS
v.LSM,v.STEP = LSM2,4
lsm.transfer_tags(LSMREF,LSM,tags="dE",tolerance=45*ARCSEC);
per_ms(jointcal_de);
v.MS = FULLMS
imager.make_image(dirty=False,stokes="IV",restore=dict(npix=NPIX,threshold=CLEAN_THRESH[1],wprojplanes=128),restore_lsm=False);
info("########## adding clean components to LSM");
ff = pyfits.open(imager.MODEL_IMAGE);
dd = ff[0].data;
dd *= 1.0769 # scale up to compensate for selfcal flux suppression
# dd[dd<0] = 0; # remove negative components
ff.writeto(LSM_CCMODEL,clobber=True);
# add model image to LSM
lsm.tigger_convert("$LSM $LSM3 --add-brick=ccmodel:$LSM_CCMODEL:2 -f");
if 5. in STEPS:
info("########## step 5: re-running DD solutions");
v.MS = FULLMS
v.LSM,v.STEP = LSM3,5
per_ms(jointcal_de_final);
if 5.5 in STEPS:
info("########## step 5.5: making joint image");
v.MS = FULLMS
v.LSM,v.STEP = LSM3,5
imager.make_image(dirty=False,stokes="IQUV",restore=dict(npix=NPIX,threshold=CLEAN_THRESH[2],wprojplanes=128),restore_lsm=True);
if 6. in STEPS:
info("########## step 6: noise sim");
per_ms(lambda:makecube(stokes="IQUV"));
v.LSM,v.STEP = LSM3,5
v.MS = FULLMS;
makecube(stokes="IQUV");
makenoise();
def jointcal(goto_step=1, last_step=10, lsmbase=None, STEPS=None):
"""Calibration for joint C and D-config data"""
info(
">>>>>>>>>>>>> output directory is $OUTDIR. Please set OUTDIR explicitly to override"
)
# setup LSM filenames based on the full MS
# note that these get interpolated once and for all here (and the _Template definitions above
# get cancelled due to the explicit assignment here). The reason for doing it like this
# is because I don't want these names to be changing due to the templates every time the
# MS changes in a per(MS) call.
v.FULLMS = MS
LSM1 = II("$DESTDIR/$LSMBASE$SUFFIX+pybdsm.lsm.html")
LSM2 = II("$DESTDIR/$LSMBASE$SUFFIX+pybdsm2.lsm.html")
LSM3 = II("$DESTDIR/$LSMBASE$SUFFIX+pybdsm2+cc.lsm.html")
LSM_CCMODEL = II("$DESTDIR/$LSMBASE$SUFFIX+ccmodel.fits")
saveconf()
stefcal.STEFCAL_DIFFGAIN_SMOOTHING = DE_SMOOTHING if not DE_INTERVALS else None
stefcal.STEFCAL_DIFFGAIN_INTERVALS = DE_INTERVALS
if DE_INTERVALS and TILE:
stefcal.STEFCAL_TDLOPTS = "ms_sel.tile_size=%d" % (TILE *
DE_INTERVALS[0])
# make MS list from sub-MSs
import glob
v.MS_List = glob.glob(MS + "/SUBMSS/*MS")
info("MS list is $MS_List")
if not MS_List:
abort("No sub-MSs found")
imager.npix = NPIX = 4096
imager.cellsize = "2arcsec"
imager.wprojplanes = 128
imager.CLEAN_ALGORITHM = "csclean"
v.LSMREF = "${MS:BASE}.refmodel.lsm.html"
THRESH_PIX, THRESH_ISL = (50, 10), (15, 5)
CLEAN_THRESH = ".4mJy", ".1mJy", ".05mJy"
stefcal.STEFCAL_STEP_INCR = 0 # precvent stefcal from auto-incrementing v.STEP: we set the step counter explicitly here
if STEPS is None:
STEPS = list(numpy.arange(goto_step, last_step + .1, .5))
STEPS = map(float, STEPS)
if STEPS[0] != 1:
info("########## restarting calibration from step %.1f" % STEPS[0])
if lsmbase:
LSMBASE = lsmbase
## initial calibration
if 1. in STEPS:
info("########## step 1: solving for G with initial LSM")
v.LSM, v.STEP = LSM0, 1
per_ms(jointcal_g)
if 1.5 in STEPS:
info("########## step 1.5: making joint image")
v.LSM, v.STEP = LSM0, 1
v.MS = FULLMS
# initial model is total flux only, made from a 2x size image to catch distant sources
imager.make_image(dirty=False,
stokes="I",
restore=dict(npix=NPIX * 2,
threshold=CLEAN_THRESH[0],
wprojplanes=128),
restore_lsm=False)
info("########## running source finder and updating model")
## now run pybdsm on restored image, output LSM will be given by variable cal.PYBDSM_OUTPUT
### NB: select on radius to exclude any artefacts picked up around 3C147 itself
lsm.pybdsm_search(thresh_pix=THRESH_PIX[0],
thresh_isl=THRESH_ISL[0],
select="r.gt.30s",
pol=False)
### merge new sources into sky model, give it a new name ($LSM1)
lsm.tigger_convert("$LSM -a ${lsm.PYBDSM_OUTPUT} $LSM1 --rename -f")
# if 2. in STEPS:
# info("########## step 2: repeating G solution");
# v.LSM,v.STEP = LSM1,2
# v.MS = FULLMS
# per_ms(jointcal_g);
if 2. in STEPS:
info("########## step 2: initial dE solution")
v.LSM, v.STEP = LSM1, 2
v.MS = FULLMS
# now, set dE tags on sources
lsm.transfer_tags(LSMREF, LSM, tags="dE", tolerance=45 * ARCSEC)
per_ms(jointcal_de_reset)
if 3. in STEPS:
info("########## step 3: re-solving for G to apply IFR solutions")
v.LSM, v.STEP = LSM1, 3
v.MS = FULLMS
per_ms(jointcal_de_apply)
info("########## running source finder and updating model")
v.MS = FULLMS
imager.make_image(dirty=False,
stokes="IV",
restore=dict(npix=NPIX,
threshold=CLEAN_THRESH[1],
wprojplanes=128),
restore_lsm=False)
## now run pybdsm on restored image, output LSM will be given by variable cal.PYBDSM_OUTPUT
### NB: select on radius to exclude any artefacts picked up around 3C147 itself
lsm.pybdsm_search(thresh_pix=THRESH_PIX[1],
thresh_isl=THRESH_ISL[1],
select="r.gt.30s")
### merge new sources into sky model, give it a new name ($LSM1)
lsm.tigger_convert("$LSM -a ${lsm.PYBDSM_OUTPUT} $LSM2 --rename -f")
if 4. in STEPS:
info(
"########## step 4: solving for G+dE with updated LSM (initial+pybdsm^2)"
)
v.MS = FULLMS
v.LSM, v.STEP = LSM2, 4
lsm.transfer_tags(LSMREF, LSM, tags="dE", tolerance=45 * ARCSEC)
per_ms(jointcal_de)
v.MS = FULLMS
imager.make_image(dirty=False,
stokes="IV",
restore=dict(npix=NPIX,
threshold=CLEAN_THRESH[1],
wprojplanes=128),
restore_lsm=False)
info("########## adding clean components to LSM")
ff = pyfits.open(imager.MODEL_IMAGE)
dd = ff[0].data
dd *= 1.0769 # scale up to compensate for selfcal flux suppression
# dd[dd<0] = 0; # remove negative components
ff.writeto(LSM_CCMODEL, clobber=True)
# add model image to LSM
lsm.tigger_convert("$LSM $LSM3 --add-brick=ccmodel:$LSM_CCMODEL:2 -f")
if 5. in STEPS:
info("########## step 5: re-running DD solutions")
v.MS = FULLMS
v.LSM, v.STEP = LSM3, 5
per_ms(jointcal_de_final)
if 5.5 in STEPS:
info("########## step 5.5: making joint image")
v.MS = FULLMS
v.LSM, v.STEP = LSM3, 5
imager.make_image(dirty=False,
stokes="IQUV",
restore=dict(npix=NPIX,
threshold=CLEAN_THRESH[2],
wprojplanes=128),
restore_lsm=True)
if 6. in STEPS:
info("########## step 6: noise sim")
per_ms(lambda: makecube(stokes="IQUV"))
v.LSM, v.STEP = LSM3, 5
v.MS = FULLMS
makecube(stokes="IQUV")
makenoise()
def selfcal_loop_pipeline(goto_step=0.):
if not os.path.isdir(v.DESTDIR):
os.makedirs(v.DESTDIR)
#copy data to corrected data column to run clean
if goto_step <= 0.:
v.STEP=1
info("########## adding columns %i"%v.STEP)
for msFile in sorted(v.MS_List):pyrap.tables.addImagingColumns(msFile)
if goto_step <= 0.5:
v.STEP=1
info("########## copying data to corrected data column step %i"%v.STEP)
pper("MS",copy_data_to_corrected_data)
#run a basic clean to perform source finding on
if goto_step <= 1:
v.STEP=1
info("########## making initial clean image step %i"%v.STEP)
wscleanDict={
'name':'wsclean',
'datacolumn':'CORRECTED_DATA',
'size': '1024 1024',
#'scale': 330./3600.,
'scale': 0.11,
'niter': 250,
'threshold': 1.,
'pol': 'I',
#'weight': 'natural',
'weight': 'briggs 0.0',
'nwlayers':32,
'channelrange': '%i %i'%(startChan,endChan),
'channelsout': 1}
for msFile in sorted(v.MS_List):
msBase=msFile.split('/')[-1].split('.MS')[0]
wscleanDict['name']=msBase+'_s%i'%v.STEP
run_wsclean(msFile,wscleanDict)
fitsfiles=glob.glob('*.fits')
for ff in fitsfiles: shutil.move(ff,v.DESTDIR)
#derive an initial sky model from the clean'd image
if goto_step <= 1.5:
v.STEP=1
fitsfiles=glob.glob(v.DESTDIR+'/*_s1-image.fits')
for ff in sorted(fitsfiles):
olsm=ff.split('-image.fits')[0]+'.lsm'
lsm.pybdsm_search(image=ff,output=olsm,thresh_pix=20,thresh_isl=15)
lsm.tigger_convert(olsm)
#run stefcal using the initial sky model
if goto_step <= 2:
v.STEP=2
lsm0files=glob.glob(v.DESTDIR+'/*_s1.lsm.html')
wscleanDict={
'name':'wsclean',
'datacolumn':'CORRECTED_DATA',
'size': '1024 1024',
#'scale': 330./3600.,
'scale': 0.11,
'niter': 500,
'threshold': 1.,
'pol': 'I',
#'weight': 'natural',
'weight': 'briggs 0.0',
'nwlayers':32,
'channelrange': '%i %i'%(startChan,endChan),
'channelsout': 1}
for msFile in sorted(v.MS_List):
msBase=msFile.split('/')[-1].split('.MS')[0]
lsm0=filter(lambda x: x.split('/')[-1].startswith(msBase),lsm0files)[0]
v.LSM=lsm0
v.MS=msFile
stefcal.STEFCAL_STEP_INCR = 0 # we set the step counter explicitly
info("########## initial calibration of %s using sky model %s step %i"%(msFile,lsm0,v.STEP))
stefcal.stefcal(output='CORR_DATA',dirty=False,restore=False,section='calico-stefcal')
info("########## making clean image step %i"%v.STEP)
wscleanDict['name']=msBase+'_s%i'%v.STEP
run_wsclean(msFile,wscleanDict)
fitsfiles=glob.glob('*.fits')
for ff in fitsfiles: shutil.move(ff,v.DESTDIR)
#derive an second sky model from the clean'd image
if goto_step <= 2.5:
v.STEP=2
fitsfiles=glob.glob(v.DESTDIR+'/*_s2-image.fits')
for ff in sorted(fitsfiles):
olsm=ff.split('-image.fits')[0]+'.lsm'
lsm.pybdsm_search(image=ff,output=olsm,thresh_pix=15,thresh_isl=10)
lsm.tigger_convert(olsm)
#run stefcal using the second sky model
if goto_step <= 3:
v.STEP=3
lsm1files=glob.glob(v.DESTDIR+'/*_s2.lsm.html')
wscleanDict={
'name':'wsclean',
'datacolumn':'CORRECTED_DATA',
'size': '1024 1024',
#'scale': 330./3600.,
'scale': 0.11,
'niter': 1000,
'threshold': 1.,
'pol': 'I',
#'weight': 'natural',
'weight': 'briggs 0.0',
'nwlayers':32,
'channelrange': '%i %i'%(startChan,endChan),
'channelsout': 1}
for msFile in sorted(v.MS_List):
x.sh('export LC_NUMERIC=en_GB.utf8')
msBase=msFile.split('/')[-1].split('.MS')[0]
lsm1=filter(lambda x: x.split('/')[-1].startswith(msBase),lsm1files)[0]
v.LSM=lsm1
v.MS=msFile
stefcal.STEFCAL_STEP_INCR = 0 # we set the step counter explicitly
info("########## second calibration of %s using sky model %s step %i"%(msFile,lsm1,v.STEP))
stefcal.stefcal(output='CORR_DATA',dirty=False,restore=False,section='calico-stefcal')
info("########## making clean image step %i"%v.STEP)
wscleanDict['name']=msBase+'_s%i'%v.STEP
run_wsclean(msFile,wscleanDict)
fitsfiles=glob.glob('*.fits')
for ff in fitsfiles: shutil.move(ff,v.DESTDIR)
#derive an sky model from the clean'd image
if goto_step <= 3.5:
v.STEP=3
fitsfiles=glob.glob(v.DESTDIR+'/*_s3-image.fits')
for ff in sorted(fitsfiles):
olsm=ff.split('-image.fits')[0]+'.lsm'
lsm.pybdsm_search(image=ff,output=olsm,thresh_pix=12,thresh_isl=5)
lsm.tigger_convert(olsm)
#run stefcal using the third sky model and produce final complex, multi-frequency images
if goto_step <= 4:
v.STEP=4
lsm2files=glob.glob(v.DESTDIR+'/*_s3.lsm.html')
wscleanDict={
'name':'wsclean',
'datacolumn':'CORRECTED_DATA',
'size': '1024 1024',
#'scale': 330./3600.,
'scale': 0.11,
'niter': 1000,
'threshold': 1.,
'pol': 'xx,xy,yx,yy',
#'weight': 'natural',
'weight': 'briggs 0.0',
'nwlayers':32,
'channelrange': '%i %i'%(startChan,endChan),
'channelsout': nsubbands,
'joinpolarizations': None}
for msFile in sorted(v.MS_List):
x.sh('export LC_NUMERIC=en_GB.utf8')
msBase=msFile.split('/')[-1].split('.MS')[0]
lsm2=filter(lambda x: x.split('/')[-1].startswith(msBase),lsm2files)[0]
v.LSM=lsm2
v.MS=msFile
stefcal.STEFCAL_STEP_INCR = 0 # we set the step counter explicitly
info("########## second calibration of %s using sky model %s step %i"%(msFile,lsm2,v.STEP))
stefcal.stefcal(output='CORR_DATA',dirty=False,restore=False,section='calico-stefcal')
wscleanDict['name']=msBase+'_s%i'%v.STEP
run_wsclean(msFile,wscleanDict)
fitsfiles=glob.glob('*.fits')
for ff in fitsfiles: shutil.move(ff,v.DESTDIR)
#clean up
if goto_step <= 5:
gaulfiles=glob.glob('*.gaul')
for gf in gaulfiles: shutil.move(gf,v.DESTDIR)
def phase_precal_selfcal_pipeline(goto_step=3.):
useStep = 1
skipCalMSlist = [
'zen.2455819.25927.uvcRREM', 'zen.2455819.27319.uvcRREM',
'zen.2455819.32190.uvcRREM', 'zen.2455819.34278.uvcRREM',
'zen.2455819.34974.uvcRREM', 'zen.2455819.38454.uvcRREM',
'zen.2455819.40542.uvcRREM', 'zen.2455819.41238.uvcRREM',
'zen.2455819.41934.uvcRREM', 'zen.2455819.42630.uvcRREM',
'zen.2455819.73947.uvcRREM', 'zen.2455819.74643.uvcRREM',
'zen.2455819.75339.uvcRREM', 'zen.2455819.76035.uvcRREM',
'zen.2455819.78819.uvcRREM', 'zen.2455819.79515.uvcRREM',
'zen.2455819.80211.uvcRREM', 'zen.2455819.80906.uvcRREM'
] #MS files to skip calibration on
skipCleanMSlist = ['zen.2455819.79515.uvcRREM'
] #problematic MS files when imaging multi-freq
if not os.path.isdir(v.DESTDIR):
os.makedirs(v.DESTDIR)
if goto_step <= 0.:
v.STEP = 1
info("########## adding columns %i" % v.STEP)
for msFile in sorted(v.MS_List):
pyrap.tables.addImagingColumns(msFile)
#copy data to corrected data column to run clean
if goto_step <= 0.5:
v.STEP = 1
info("########## copying data to corrected data column step %i" %
v.STEP)
pper("MS", copy_data_to_corrected_data)
#run a basic clean to perform source finding on
if goto_step <= 1:
v.STEP = 1
info("########## making initial clean image step %i" % v.STEP)
wscleanDict = {
'name': 'wsclean',
'datacolumn': 'CORRECTED_DATA',
'size': '1024 1024',
#'scale': 330./3600.,
'scale': 0.11,
'niter': 250,
'threshold': 100.,
'pol': 'I',
#'weight': 'natural',
'weight': 'briggs 0.0',
'nwlayers': 32,
'channelrange': '%i %i' % (startChan, endChan),
'channelsout': 1
}
for msFile in sorted(v.MS_List):
msBase = msFile.split('/')[-1].split('.MS')[0]
wscleanDict['name'] = msBase + '_s%i' % v.STEP
run_wsclean(msFile, wscleanDict)
fitsfiles = glob.glob('*.fits')
for ff in fitsfiles:
shutil.move(ff, v.DESTDIR)
#derive a shallow depth sky model from the clean'd image
if goto_step <= 1.5:
v.STEP = 1
fitsfiles = glob.glob(v.DESTDIR + '/*_s1-image.fits')
for ff in sorted(fitsfiles):
olsm = ff.split('-image.fits')[0] + '.lsm'
lsm.pybdsm_search(image=ff,
output=olsm,
thresh_pix=20,
thresh_isl=15)
lsm.tigger_convert(olsm)
#run stefcal using the sky model, produce a Stokes I image for source finding
if goto_step <= 2:
v.STEP = 2
lsm0files = glob.glob(v.DESTDIR + '/*_s1.lsm.html')
wscleanDict = {
'name': 'wsclean',
'datacolumn': 'CORRECTED_DATA',
'size': '1024 1024',
#'scale': 330./3600.,
'scale': 0.11,
'niter': 250,
'threshold': 100.,
'pol': 'I',
#'weight': 'natural',
'weight': 'briggs 0.0',
'nwlayers': 32,
'channelrange': '%i %i' % (startChan, endChan),
'channelsout': 1
}
for msFile in sorted(v.MS_List):
msBase = msFile.split('/')[-1].split('.MS')[0]
if msBase in skipCalMSlist:
info('Skipping this MS file')
else:
x.sh('export LC_NUMERIC=en_GB.utf8')
lsm0 = filter(lambda x: x.split('/')[-1].startswith(msBase),
lsm0files)[0]
v.LSM = lsm0
v.MS = msFile
stefcal.STEFCAL_STEP_INCR = 0 # we set the step counter explicitly
info(
"########## initial calibration of %s using sky model %s step %i"
% (msFile, lsm0, v.STEP))
stefcal.stefcal(output='CORR_DATA',
dirty=False,
restore=False,
section='calico-stefcal')
info("########## making clean image step %i" % v.STEP)
wscleanDict['name'] = msBase + '_s%i' % v.STEP
run_wsclean(msFile, wscleanDict)
fitsfiles = glob.glob('*.fits')
for ff in fitsfiles:
shutil.move(ff, v.DESTDIR)
#derive a deeper depth sky model from the clean'd image
if goto_step <= 2.5:
v.STEP = 2
fitsfiles = glob.glob(v.DESTDIR + '/*_s2-image.fits')
for ff in sorted(fitsfiles):
olsm = ff.split('-image.fits')[0] + '.lsm'
lsm.pybdsm_search(image=ff,
output=olsm,
thresh_pix=10,
thresh_isl=7)
lsm.tigger_convert(olsm)
#generate multi-frequency, complex images for image domain corrections
if goto_step <= 3:
v.STEP = 3
wscleanDict = {
'name': 'wsclean',
'datacolumn': 'CORRECTED_DATA',
'size': '1024 1024',
#'scale': 330./3600.,
'scale': 0.11,
#'niter': 1000,
'niter': 1,
'threshold': 100.,
'pol': 'xx,xy,yx,yy',
'weight': 'natural',
#'weight': 'briggs 0.0',
'nwlayers': 32,
'channelrange': '%i %i' % (startChan, endChan),
'channelsout': nsubbands,
'joinpolarizations': None
}
for msFile in sorted(v.MS_List):
x.sh('export LC_NUMERIC=en_GB.utf8')
msBase = msFile.split('/')[-1].split('.MS')[0]
if msBase in skipCleanMSlist: wscleanDict['niter'] = 1
else: wscleanDict['niter'] = 1
#else: wscleanDict['niter']=1000
info("########## making clean image step %i" % v.STEP)
wscleanDict['name'] = msBase + '_s%i' % v.STEP
run_wsclean(msFile, wscleanDict)
fitsfiles = glob.glob('*.fits')
for ff in fitsfiles:
shutil.move(ff, v.DESTDIR)
def cal_ms():
"""cal_ms: runs a calibration loop over the current MS"""
# initialize the calibration "step" counter and "label". These are used
# to auto-generate output filenames; cal.stefcal() below automatically increments
# cal.STEP
v.STEP = 0
# set the superglobal LSM variable. See explanation for "v." in the text
v.LSM = LSM0
# info(), warn() and abort() are Pyxis functions for writing output to the log
info("########## solving for G with initial LSM")
# do one stefcal run with the initial model, produce corrected residuals,
# do simple flagging on the residuals. See cal.stefcal() for full docs.
stefcal.stefcal(stefcal_reset_all=True,
output="CORR_RES",
restore=False,
flag_threshold=(1, .5))
# If we got to this point, then assume things are rolling along fine.
# Copy the recipe and config to the destination directory for future reference
# (very useful when modifying recipes, so you can keep track of what settings
# are responsible for what output!)
xo.sh("cp pyxis*.py pyxis*.conf ${mqt.TDLCONFIG} $DESTDIR", shell=True)
info(
"########## remaking data image, running source finder and updating model"
)
# apply previous stefcal solutions to produce a corrected data image.
# Make a restored image with 2000 CLEAN iterations.
stefcal.stefcal(apply_only=True,
output="CORR_DATA",
plotvis=False,
restore=dict(niter=2000),
restore_lsm=False)
## now run pybdsm on the restored image, write output to a new LSM file
lsm.pybdsm_search(threshold=5, output=LSM1)
# the new sky model becomes the "current" LSM
v.LSM = LSM1
# note that in principle this step is not needed (may as well go on directly
# to the dE solutions, below), but it is instructive, as it will produce an intermediate-stage
# image. But to save time, set restore=False to skip making clean images
info("########## recalibrating with new model")
stefcal.stefcal(stefcal_reset_all=True,
output="CORR_RES",
restore=False,
flag_threshold=(1, .5))
# if we have a "reference" sky model configured, use it to set dE tags on
# sources in our new sky model. Sources with dE tags will have DDE solutions.
# The cal.transfer_tags() function sets the specified tag on any source
# near enough to a reference model source with that tag.
if LSMREF:
lsm.transfer_tags(LSMREF, LSM, tags="dE", tolerance=45 * ARCSEC)
info("########## recalibrating with dEs")
# re-run stefcal with the new model (and with direction dependent
# solutions on dE-tagged sources, if any). Make a corrected residual
# image, run CLEAN on it, restore the LSM sources back into the
# image
stefcal.stefcal(stefcal_reset_all=True,
diffgains=True,
output="CORR_RES",
restore=dict(niter=1000),
restore_lsm=True,
flag_threshold=(1, .5))
# no reference LSM? then still need to make a restored image from the step above
else:
imager.make_image(dirty=False,
restore=dict(niter=1000),
restore_lsm=True)
# the resulting image filename is in cal.FULLREST_IMAGE,
# add it to our image list file
IMAGE_LIST.add("${imager.FULLREST_IMAGE}")
def phase_precal_selfcal_pipeline(goto_step=3.):
useStep=1
skipCalMSlist=[ 'zen.2455819.25927.uvcRREM',
'zen.2455819.27319.uvcRREM',
'zen.2455819.32190.uvcRREM',
'zen.2455819.34278.uvcRREM',
'zen.2455819.34974.uvcRREM',
'zen.2455819.38454.uvcRREM',
'zen.2455819.40542.uvcRREM',
'zen.2455819.41238.uvcRREM',
'zen.2455819.41934.uvcRREM',
'zen.2455819.42630.uvcRREM',
'zen.2455819.73947.uvcRREM',
'zen.2455819.74643.uvcRREM',
'zen.2455819.75339.uvcRREM',
'zen.2455819.76035.uvcRREM',
'zen.2455819.78819.uvcRREM',
'zen.2455819.79515.uvcRREM',
'zen.2455819.80211.uvcRREM',
'zen.2455819.80906.uvcRREM'] #MS files to skip calibration on
skipCleanMSlist=['zen.2455819.79515.uvcRREM'] #problematic MS files when imaging multi-freq
if not os.path.isdir(v.DESTDIR):
os.makedirs(v.DESTDIR)
if goto_step <= 0.:
v.STEP=1
info("########## adding columns %i"%v.STEP)
for msFile in sorted(v.MS_List):pyrap.tables.addImagingColumns(msFile)
#copy data to corrected data column to run clean
if goto_step <= 0.5:
v.STEP=1
info("########## copying data to corrected data column step %i"%v.STEP)
pper("MS",copy_data_to_corrected_data)
#run a basic clean to perform source finding on
if goto_step <= 1:
v.STEP=1
info("########## making initial clean image step %i"%v.STEP)
wscleanDict={
'name':'wsclean',
'datacolumn':'CORRECTED_DATA',
'size': '1024 1024',
#'scale': 330./3600.,
'scale': 0.11,
'niter': 250,
'threshold': 100.,
'pol': 'I',
#'weight': 'natural',
'weight': 'briggs 0.0',
'nwlayers':32,
'channelrange': '%i %i'%(startChan,endChan),
'channelsout': 1}
for msFile in sorted(v.MS_List):
msBase=msFile.split('/')[-1].split('.MS')[0]
wscleanDict['name']=msBase+'_s%i'%v.STEP
run_wsclean(msFile,wscleanDict)
fitsfiles=glob.glob('*.fits')
for ff in fitsfiles: shutil.move(ff,v.DESTDIR)
#derive a shallow depth sky model from the clean'd image
if goto_step <= 1.5:
v.STEP=1
fitsfiles=glob.glob(v.DESTDIR+'/*_s1-image.fits')
for ff in sorted(fitsfiles):
olsm=ff.split('-image.fits')[0]+'.lsm'
lsm.pybdsm_search(image=ff,output=olsm,thresh_pix=20,thresh_isl=15)
lsm.tigger_convert(olsm)
#run stefcal using the sky model, produce a Stokes I image for source finding
if goto_step <= 2:
v.STEP=2
lsm0files=glob.glob(v.DESTDIR+'/*_s1.lsm.html')
wscleanDict={
'name':'wsclean',
'datacolumn':'CORRECTED_DATA',
'size': '1024 1024',
#'scale': 330./3600.,
'scale': 0.11,
'niter': 250,
'threshold': 100.,
'pol': 'I',
#'weight': 'natural',
'weight': 'briggs 0.0',
'nwlayers':32,
'channelrange': '%i %i'%(startChan,endChan),
'channelsout': 1}
for msFile in sorted(v.MS_List):
msBase=msFile.split('/')[-1].split('.MS')[0]
if msBase in skipCalMSlist:
info('Skipping this MS file')
else:
x.sh('export LC_NUMERIC=en_GB.utf8')
lsm0=filter(lambda x: x.split('/')[-1].startswith(msBase),lsm0files)[0]
v.LSM=lsm0
v.MS=msFile
stefcal.STEFCAL_STEP_INCR = 0 # we set the step counter explicitly
info("########## initial calibration of %s using sky model %s step %i"%(msFile,lsm0,v.STEP))
stefcal.stefcal(output='CORR_DATA',dirty=False,restore=False,section='calico-stefcal')
info("########## making clean image step %i"%v.STEP)
wscleanDict['name']=msBase+'_s%i'%v.STEP
run_wsclean(msFile,wscleanDict)
fitsfiles=glob.glob('*.fits')
for ff in fitsfiles: shutil.move(ff,v.DESTDIR)
#derive a deeper depth sky model from the clean'd image
if goto_step <= 2.5:
v.STEP=2
fitsfiles=glob.glob(v.DESTDIR+'/*_s2-image.fits')
for ff in sorted(fitsfiles):
olsm=ff.split('-image.fits')[0]+'.lsm'
lsm.pybdsm_search(image=ff,output=olsm,thresh_pix=10,thresh_isl=7)
lsm.tigger_convert(olsm)
#generate multi-frequency, complex images for image domain corrections
if goto_step <= 3:
v.STEP=3
wscleanDict={
'name':'wsclean',
'datacolumn':'CORRECTED_DATA',
'size': '1024 1024',
#'scale': 330./3600.,
'scale': 0.11,
#'niter': 1000,
'niter': 1,
'threshold': 100.,
'pol': 'xx,xy,yx,yy',
'weight': 'natural',
#'weight': 'briggs 0.0',
'nwlayers':32,
'channelrange': '%i %i'%(startChan,endChan),
'channelsout': nsubbands,
'joinpolarizations': None}
for msFile in sorted(v.MS_List):
x.sh('export LC_NUMERIC=en_GB.utf8')
msBase=msFile.split('/')[-1].split('.MS')[0]
if msBase in skipCleanMSlist: wscleanDict['niter']=1
else: wscleanDict['niter']=1
#else: wscleanDict['niter']=1000
info("########## making clean image step %i"%v.STEP)
wscleanDict['name']=msBase+'_s%i'%v.STEP
run_wsclean(msFile,wscleanDict)
fitsfiles=glob.glob('*.fits')
for ff in fitsfiles: shutil.move(ff,v.DESTDIR)
def selfcal_loop_pipeline(goto_step=0.):
if not os.path.isdir(v.DESTDIR):
os.makedirs(v.DESTDIR)
#copy data to corrected data column to run clean
if goto_step <= 0.:
v.STEP = 1
info("########## adding columns %i" % v.STEP)
for msFile in sorted(v.MS_List):
pyrap.tables.addImagingColumns(msFile)
if goto_step <= 0.5:
v.STEP = 1
info("########## copying data to corrected data column step %i" %
v.STEP)
pper("MS", copy_data_to_corrected_data)
#run a basic clean to perform source finding on
if goto_step <= 1:
v.STEP = 1
info("########## making initial clean image step %i" % v.STEP)
wscleanDict = {
'name': 'wsclean',
'datacolumn': 'CORRECTED_DATA',
'size': '1024 1024',
#'scale': 330./3600.,
'scale': 0.11,
'niter': 250,
'threshold': 1.,
'pol': 'I',
#'weight': 'natural',
'weight': 'briggs 0.0',
'nwlayers': 32,
'channelrange': '%i %i' % (startChan, endChan),
'channelsout': 1
}
for msFile in sorted(v.MS_List):
msBase = msFile.split('/')[-1].split('.MS')[0]
wscleanDict['name'] = msBase + '_s%i' % v.STEP
run_wsclean(msFile, wscleanDict)
fitsfiles = glob.glob('*.fits')
for ff in fitsfiles:
shutil.move(ff, v.DESTDIR)
#derive an initial sky model from the clean'd image
if goto_step <= 1.5:
v.STEP = 1
fitsfiles = glob.glob(v.DESTDIR + '/*_s1-image.fits')
for ff in sorted(fitsfiles):
olsm = ff.split('-image.fits')[0] + '.lsm'
lsm.pybdsm_search(image=ff,
output=olsm,
thresh_pix=20,
thresh_isl=15)
lsm.tigger_convert(olsm)
#run stefcal using the initial sky model
if goto_step <= 2:
v.STEP = 2
lsm0files = glob.glob(v.DESTDIR + '/*_s1.lsm.html')
wscleanDict = {
'name': 'wsclean',
'datacolumn': 'CORRECTED_DATA',
'size': '1024 1024',
#'scale': 330./3600.,
'scale': 0.11,
'niter': 500,
'threshold': 1.,
'pol': 'I',
#'weight': 'natural',
'weight': 'briggs 0.0',
'nwlayers': 32,
'channelrange': '%i %i' % (startChan, endChan),
'channelsout': 1
}
for msFile in sorted(v.MS_List):
msBase = msFile.split('/')[-1].split('.MS')[0]
lsm0 = filter(lambda x: x.split('/')[-1].startswith(msBase),
lsm0files)[0]
v.LSM = lsm0
v.MS = msFile
stefcal.STEFCAL_STEP_INCR = 0 # we set the step counter explicitly
info(
"########## initial calibration of %s using sky model %s step %i"
% (msFile, lsm0, v.STEP))
stefcal.stefcal(output='CORR_DATA',
dirty=False,
restore=False,
section='calico-stefcal')
info("########## making clean image step %i" % v.STEP)
wscleanDict['name'] = msBase + '_s%i' % v.STEP
run_wsclean(msFile, wscleanDict)
fitsfiles = glob.glob('*.fits')
for ff in fitsfiles:
shutil.move(ff, v.DESTDIR)
#derive an second sky model from the clean'd image
if goto_step <= 2.5:
v.STEP = 2
fitsfiles = glob.glob(v.DESTDIR + '/*_s2-image.fits')
for ff in sorted(fitsfiles):
olsm = ff.split('-image.fits')[0] + '.lsm'
lsm.pybdsm_search(image=ff,
output=olsm,
thresh_pix=15,
thresh_isl=10)
lsm.tigger_convert(olsm)
#run stefcal using the second sky model
if goto_step <= 3:
v.STEP = 3
lsm1files = glob.glob(v.DESTDIR + '/*_s2.lsm.html')
wscleanDict = {
'name': 'wsclean',
'datacolumn': 'CORRECTED_DATA',
'size': '1024 1024',
#'scale': 330./3600.,
'scale': 0.11,
'niter': 1000,
'threshold': 1.,
'pol': 'I',
#'weight': 'natural',
'weight': 'briggs 0.0',
'nwlayers': 32,
'channelrange': '%i %i' % (startChan, endChan),
'channelsout': 1
}
for msFile in sorted(v.MS_List):
x.sh('export LC_NUMERIC=en_GB.utf8')
msBase = msFile.split('/')[-1].split('.MS')[0]
lsm1 = filter(lambda x: x.split('/')[-1].startswith(msBase),
lsm1files)[0]
v.LSM = lsm1
v.MS = msFile
stefcal.STEFCAL_STEP_INCR = 0 # we set the step counter explicitly
info(
"########## second calibration of %s using sky model %s step %i"
% (msFile, lsm1, v.STEP))
stefcal.stefcal(output='CORR_DATA',
dirty=False,
restore=False,
section='calico-stefcal')
info("########## making clean image step %i" % v.STEP)
wscleanDict['name'] = msBase + '_s%i' % v.STEP
run_wsclean(msFile, wscleanDict)
fitsfiles = glob.glob('*.fits')
for ff in fitsfiles:
shutil.move(ff, v.DESTDIR)
#derive an sky model from the clean'd image
if goto_step <= 3.5:
v.STEP = 3
fitsfiles = glob.glob(v.DESTDIR + '/*_s3-image.fits')
for ff in sorted(fitsfiles):
olsm = ff.split('-image.fits')[0] + '.lsm'
lsm.pybdsm_search(image=ff,
output=olsm,
thresh_pix=12,
thresh_isl=5)
lsm.tigger_convert(olsm)
#run stefcal using the third sky model and produce final complex, multi-frequency images
if goto_step <= 4:
v.STEP = 4
lsm2files = glob.glob(v.DESTDIR + '/*_s3.lsm.html')
wscleanDict = {
'name': 'wsclean',
'datacolumn': 'CORRECTED_DATA',
'size': '1024 1024',
#'scale': 330./3600.,
'scale': 0.11,
'niter': 1000,
'threshold': 1.,
'pol': 'xx,xy,yx,yy',
#'weight': 'natural',
'weight': 'briggs 0.0',
'nwlayers': 32,
'channelrange': '%i %i' % (startChan, endChan),
'channelsout': nsubbands,
'joinpolarizations': None
}
for msFile in sorted(v.MS_List):
x.sh('export LC_NUMERIC=en_GB.utf8')
msBase = msFile.split('/')[-1].split('.MS')[0]
lsm2 = filter(lambda x: x.split('/')[-1].startswith(msBase),
lsm2files)[0]
v.LSM = lsm2
v.MS = msFile
stefcal.STEFCAL_STEP_INCR = 0 # we set the step counter explicitly
info(
"########## second calibration of %s using sky model %s step %i"
% (msFile, lsm2, v.STEP))
stefcal.stefcal(output='CORR_DATA',
dirty=False,
restore=False,
section='calico-stefcal')
wscleanDict['name'] = msBase + '_s%i' % v.STEP
run_wsclean(msFile, wscleanDict)
fitsfiles = glob.glob('*.fits')
for ff in fitsfiles:
shutil.move(ff, v.DESTDIR)
#clean up
if goto_step <= 5:
gaulfiles = glob.glob('*.gaul')
for gf in gaulfiles:
shutil.move(gf, v.DESTDIR)
#!/usr/bin/python
# uses pybdsm to generate respective catalogs for all our fits files
#importing Pybdsm,Pyxis and Tigger
import os
import Tigger
from lofar import bdsm
import Pyxis
import lsm
#storing all the fits files in a list
filenames = ['PKS1510-089_20140110_hh.fits','PKS1510-089_20140112_hh.fits','PKS1510-089_20140501_hh.fits','PKS1510-089_20140521_hh.fits','PKS1510-089_20140524_hh.fits','PKS1510-089_20140603_hh.fits','PKS1510-089_20140614_hh.fits','PKS1510-089_20140622_hh.fits','PKS1510-089_20140625_hh.fits']
i = 0
#processing the images using bdsm
for f in filenames:
lsm.pybdsm_search(image=f,thresh_isl=3,thresh_pix = 7,clobber = True)
os.system("mv /home/ulrich/Workspace/plotsspw0/spw0-s1-_pybdsm.lsm.gaul /home/ulrich/Workspace/plotsspw0/image-%d.lsm.gaul"%(i))
os.system("mv /home/ulrich/Workspace/plotsspw0/spw0-s1-_pybdsm.lsm.html /home/ulrich/Workspace/plotsspw0/image-%d.lsm.html"%(i))
i = i + 1
