def c_cal (goto_step=1): """Calibration for C-config data""" ## initial calibration if goto_step > 1: info("########## restarting calibration from step $goto_step"); # Calibration step -- this is just a user-defined label used in filenames (etc. "blahblah_s1"), which serves to keep the output from each step # of a pipeline separate. If this is numeric, then functions such as stefcal.stefcal() will increment it automatically first thing. Otherwise you can set # it yourself to some more fancy label. Here we also provide a way to hop to particular step via goto_step v.STEP = goto_step-1; if goto_step < 2: # set the superglobal LSM v.LSM = LSM0 info("########## solving for G with initial LSM"); # no w-proj for dirty map to save time stefcal.stefcal(stefcal_reset_all=True, dirty=dict(wprojplanes=0,npix=NPIX), restore=dict(npix=NPIX,threshold=CLEAN_THRESH[0],wprojplanes=128)); info("########## running source finder and updating model"); ## now run pybdsm on restored image, output LSM will be given by variable cal.PYBDSM_OUTPUT ### NB: select on radius to exclude any artefacts picked up around 3C147 itself lsm.pybdsm_search(thresh_pix=THRESH_PIX,thresh_isl=THRESH_ISL,select="r.gt.30s"); ### merge new sources into sky model, give it a new name ($LSM1) lsm.tigger_convert("$LSM -a ${lsm.PYBDSM_OUTPUT} $LSM1 --rename -f"); if goto_step < 3: info("########## solving for G+dE with updated LSM (initial+pybdsm)"); v.LSM = LSM1 # now, set dE tags on sources lsm.transfer_tags(LSMREF,LSM,tags="dE",tolerance=45*ARCSEC); stefcal.stefcal(stefcal_reset_all=True,diffgains=True,dirty=dict(wprojplanes=0,npix=NPIX)); if goto_step < 4: info("########## re-solving for G to apply IFR solutions"); v.LSM = LSM1 stefcal.stefcal(diffgains=True,diffgain_apply_only=True, dirty=dict(wprojplanes=0,npix=NPIX), restore=dict(npix=NPIX,threshold=CLEAN_THRESH[1])); info("########## adding clean components to LSM"); ff = pyfits.open(imager.MODEL_IMAGE); dd = ff[0].data; dd *= 1.0769 # scale up to compensate for selfcal flux suppression # dd[dd<0] = 0; # remove negative components ff.writeto(LSM_CCMODEL,clobber=True); # add model image to LSM lsm.tigger_convert("$LSM $LSM2 --add-brick=ccmodel:$LSM_CCMODEL:2 -f"); if goto_step < 5: info("########## re-running DD solutions"); v.LSM = LSM2 # make final image stefcal.stefcal(dirty=dict(wprojplanes=0,npix=NPIX),diffgains=True, restore=dict(npix=NPIX,threshold=CLEAN_THRESH[2]), label="dE"); # make per-channel cube makecube(NPIX); # make noise images makenoise();
def find_sources(image='${im.RESTORED_IMAGE}', psf_image="${im.PSF_IMAGE}", thresh_isl=None, thresh_pix=None, neg_isl=None, neg_pix=None, output='$PLSM', reliability=False, config=None, outdir="$DESTDIR", gaul="$GAUL", **kw): image, psf_image, output, gaul, outdir = interpolate_locals( 'image psf_image output gaul outdir') thresh_pix = thresh_pix or THRESHOLDS[0] thresh_isl = thresh_isl or THRESHOLDS[1] neg_thresh_pix = neg_pix or NEG_THRESH[0] neg_thresh_isl = neg_isl or NEG_THRESH[1] if reliability: x.sh( "sourcery -i $image -p $psf_image -od $outdir -nisl=$neg_thresh_isl\ -npix=$neg_thresh_pix -pisl=$thresh_isl -ppix=$thresh_pix -ps=3 -ns=1\ -apsf -alv") else: lsm.pybdsm_search(image=image, output=output, thresh_pix=thresh_pix, thresh_isl=thresh_isl, **kw)
def cal_ms(lsm0='$LSM0', start=0, stop=4): """ Run pipeline on a single MS""" lsm0 = II(lsm0) # Calibrate each MS #run_cmd = lambda : calibrate(lsmname=lsm0) #run_cmd = calibrate(lsmname=lsm0) v.MS = CONCAT_MS calibrate(lsmname=lsm0) #pper("MS",run_cmd) #calibrate(lsmname=lsm0) # image combined MS #ms.virtconcat(output=CONCAT_MS) v.MS = CONCAT_MS im.make_image(restore=True, psf=True, restore_lsm=False) # run source finder lsm.pybdsm_search(thresh_pix=5 , thresh_isl=3) v.LSM = lsm.PYBDSM_OUTPUT x.sh("tigger-convert --append $LSM $LSM0 $LSMFINAL -f") # make final restored map x.sh("tigger-restore ${im.RESTORED_IMAGE} $LSM0 ${im.FULLREST_IMAGE} -f ")
def cal_DI(msname="$MS", lsm0='$LSM0', timeint=None, freqint=None, smooth=None, start=1, stop=5, **kw): """ Run pipeline on a single MS""" msname, lsm0 = interpolate_locals("msname lsm0") # Use the initial model to do the direction independent calibration. calibrate_DI(msname, lsm0) im.make_image(restore=True, psf=True, restore_lsm=False) lsm.pybdsm_search(thresh_pix=5, thresh_isl=3) v.LSM = lsm.PYBDSM_OUTPUT lsm1 = v.LSM x.sh("tigger-convert --append $LSM $lsm0 $LSMFINAL -f") x.sh( "tigger-restore ${im.RESTORED_IMAGE} $LSMFINAL ${im.FULLREST_IMAGE} -f " )
def runcal(goto_step=1): ## initial calibration if goto_step > 1: info("########## restarting calibration from step $goto_step") # Calibration step -- this is just a user-defined label used in filenames (etc. "blahblah_s1"), which serves to keep the output from each step # of a pipeline separate. If this is numeric, then functions such as stefcal.stefcal() will increment it automatically first thing. Otherwise you can set # it yourself to some more fancy label. Here we also provide a way to hop to particular step via goto_step v.STEP = goto_step - 1 if goto_step < 2: # set the superglobal LSM v.LSM = LSM0 info("########## solving for G with initial LSM") # no w-proj for dirty map to save time stefcal.stefcal(stefcal_reset_ifr_gains=True, dirty=dict(wprojplanes=0), restore=True) info("########## running source finder and updating model") ## now run pybdsm on restored image, output LSM will be given by variable cal.PYBDSM_OUTPUT lsm.pybdsm_search(threshold=7) ### merge new sources into sky model, give it a new name ($LSM1) lsm.tigger_convert("$LSM -a ${lsm.PYBDSM_OUTPUT} $LSM1 --rename -f") if goto_step < 3: info("########## solving for G with updated LSM (initial+pybdsm)") v.LSM = LSM1 stefcal.stefcal(dirty=dict(wprojplanes=0)) if goto_step < 4: info("########## re-solving for G to apply IFR solutions") stefcal.stefcal(dirty=dict(wprojplanes=0), restore=True) info("########## adding clean components to LSM") CCMODEL = II("ccmodel-ddid${ms.DDID}.fits") # note the per-style variable interpolation done by the II() function ff = pyfits.open(imager.MODEL_IMAGE) dd = ff[0].data dd *= 1.0769 # scale up to compensate for selfcal flux suppression # dd[dd<0] = 0; # remove negative components ff.writeto(CCMODEL, clobber=True) # add model image to LSM lsm.tigger_convert("$LSM $LSM2 --add-brick=ccmodel:$CCMODEL:2 -f") if goto_step < 5: info("########## solving for G with updated LSM (inital+pybdsm+cc)") v.LSM = LSM2 stefcal.stefcal(dirty=dict(wprojplanes=0)) if goto_step < 6: info("########## running DD solutions") v.LSM = LSM2 # now, set dE tags on sources lsm.transfer_tags(LSMREF, LSM, tags="dE", tolerance=45 * ARCSEC) # make final image stefcal.stefcal(dirty=dict(wprojplanes=0), diffgains=True, restore=True, label="dE")
def runcal (goto_step=1): ## initial calibration if goto_step > 1: info("########## restarting calibration from step $goto_step"); # Calibration step -- this is just a user-defined label used in filenames (etc. "blahblah_s1"), which serves to keep the output from each step # of a pipeline separate. If this is numeric, then functions such as stefcal.stefcal() will increment it automatically first thing. Otherwise you can set # it yourself to some more fancy label. Here we also provide a way to hop to particular step via goto_step v.STEP = goto_step-1; if goto_step < 2: # set the superglobal LSM v.LSM = LSM0 info("########## solving for G with initial LSM"); # no w-proj for dirty map to save time stefcal.stefcal(stefcal_reset_ifr_gains=True,dirty=dict(wprojplanes=0),restore=True); info("########## running source finder and updating model"); ## now run pybdsm on restored image, output LSM will be given by variable cal.PYBDSM_OUTPUT lsm.pybdsm_search(threshold=7); ### merge new sources into sky model, give it a new name ($LSM1) lsm.tigger_convert("$LSM -a ${lsm.PYBDSM_OUTPUT} $LSM1 --rename -f"); if goto_step < 3: info("########## solving for G with updated LSM (initial+pybdsm)"); v.LSM = LSM1 stefcal.stefcal(dirty=dict(wprojplanes=0)); if goto_step < 4: info("########## re-solving for G to apply IFR solutions"); stefcal.stefcal(dirty=dict(wprojplanes=0),restore=True); info("########## adding clean components to LSM"); CCMODEL = II("ccmodel-ddid${ms.DDID}.fits"); # note the per-style variable interpolation done by the II() function ff = pyfits.open(imager.MODEL_IMAGE); dd = ff[0].data; dd *= 1.0769 # scale up to compensate for selfcal flux suppression # dd[dd<0] = 0; # remove negative components ff.writeto(CCMODEL,clobber=True); # add model image to LSM lsm.tigger_convert("$LSM $LSM2 --add-brick=ccmodel:$CCMODEL:2 -f"); if goto_step < 5: info("########## solving for G with updated LSM (inital+pybdsm+cc)"); v.LSM = LSM2 stefcal.stefcal(dirty=dict(wprojplanes=0)); if goto_step < 6: info("########## running DD solutions"); v.LSM = LSM2 # now, set dE tags on sources lsm.transfer_tags(LSMREF,LSM,tags="dE",tolerance=45*ARCSEC); # make final image stefcal.stefcal(dirty=dict(wprojplanes=0),diffgains=True,restore=True,label="dE");
def run_all(): info( "############## Make image from the data column, Model the sky and dE tag ######################" ) im.make_image(column="DATA", restore=True, psf=True, restore_lsm=False) make_clean_model() info("######################## Calibrating G+B only ####################") cal_DI(lsm0=LSM0) im.make_image(column="CORRECTED_DATA", restore=True, psf=True, restore_lsm=False) image = im.RESTORED_IMAGE lsm.pybdsm_search(image, thresh_pix=5, thresh_isl=3) v.LSM = lsm.PYBDSM_OUTPUT lsm1 = v.LSM make_clean_model(image=im.RESTORED_IMAGE, lsm0=LSM, threshold=3) info( "################# CALIBRATING DIFFENTIAL GAIN (dE): the flyswatter (O.M.Smirnov) ###################" ) cal_DD(lsm0=LSMFINAL) im.make_image(column="CORRECTED_DATA", restore=True, psf=True, restore_lsm=False) image = im.RESTORED_IMAGE lsm.pybdsm_search(image, thresh_pix=5, thresh_isl=3) v.LSM = lsm.PYBDSM_OUTPUT lsm1 = v.LSM make_clean_model(image=im.RESTORED_IMAGE, lsm0=LSM, threshold=3) info( "############## Change the apparent flux to intrinsic flux for beam calibration ######################" ) APP_INT(lsmname=LSM0, output=LSM_INTRINSIC) info( "############## Do only G+B with beam and with beam ######################" ) cal_BM(lsm0=LSM_INTRINSIC) x.sh("tigger-restore ${im.RESTORED_IMAGE} $LSM0 ${im.FULLREST_IMAGE} -f ")
def find_sources(image='${im.RESTORED_IMAGE}',psf_image="${im.PSF_IMAGE}", thresh_isl=None, thresh_pix=None, neg_isl=None, neg_pix=None, output='$PLSM', reliability=False, config=None, outdir="$DESTDIR", gaul="$GAUL", **kw): image, psf_image, output, gaul, outdir = interpolate_locals('image psf_image output gaul outdir') thresh_pix = thresh_pix or THRESHOLDS[0] thresh_isl = thresh_isl or THRESHOLDS[1] neg_thresh_pix = neg_pix or NEG_THRESH[0] neg_thresh_isl = neg_isl or NEG_THRESH[1] if reliability: x.sh("sourcery -i $image -p $psf_image -od $outdir -nisl=$neg_thresh_isl\ -npix=$neg_thresh_pix -pisl=$thresh_isl -ppix=$thresh_pix -ps=3 -ns=1\ -apsf -alv") else: lsm.pybdsm_search(image=image, output=output, thresh_pix=thresh_pix, thresh_isl=thresh_isl, **kw)
def cal_ms(lsm0='$LSM0', start=0, stop=4): """ Run pipeline on a single MS""" lsm0 = II(lsm0) # Calibrate each MS run_cmd = lambda: calibrate(lsmname=lsm0) pper("MS", run_cmd) # image combined MS ms.virtconcat(output=CONCAT_MS) v.MS = CONCAT_MS im.make_image(restore=True, psf=True, restore_lsm=False) # run source finder lsm.pybdsm_search(thresh_pix=5, thresh_isl=3) v.LSM = lsm.PYBDSM_OUTPUT x.sh("tigger-convert --append $LSM $LSM0 $LSMFINAL -f") # make final restored map x.sh("tigger-restore ${im.RESTORED_IMAGE} $LSM0 ${im.FULLREST_IMAGE} -f ")
def make_clean_model(image="${im.RESTORED_IMAGE}", psf_image="${im.PSF_IMAGE}", lsm0="$LSM0", threshold=7): image, psf_image, lsm0 = interpolate_locals("image psf_image lsm0") lsm.pybdsm_search(image, output=lsm0, threshold=threshold) catalog = Tigger.load(lsm0) src = catalog.sources cs = [] for i in range(len(src)): position = [deg(src[i].pos.ra), deg(src[i].pos.dec)] c = correlation_factor(src[i], psf=psf_image, img=image, pos_sky=position, step=120) cs.append(c) for i in range(len(src)): if cs[i] > 0.6 * max(cs): src[i].setTag('dE', True) catalog.save(lsm0)
def cal_ms (): """cal_ms: runs a calibration loop over the current MS""" # initialize the calibration "step" counter and "label". These are used # to auto-generate output filenames; cal.stefcal() below automatically increments # cal.STEP v.STEP = 0 # set the superglobal LSM variable. See explanation for "v." in the text v.LSM = LSM0 # info(), warn() and abort() are Pyxis functions for writing output to the log info("########## solving for G with initial LSM") # do one stefcal run with the initial model, produce corrected residuals, # do simple flagging on the residuals. See cal.stefcal() for full docs. stefcal.stefcal(stefcal_reset_all=True,output="CORR_RES",restore=False,flag_threshold=(1,.5)) # If we got to this point, then assume things are rolling along fine. # Copy the recipe and config to the destination directory for future reference # (very useful when modifying recipes, so you can keep track of what settings # are responsible for what output!) xo.sh("cp pyxis*.py pyxis*.conf ${mqt.TDLCONFIG} $DESTDIR",shell=True) info("########## remaking data image, running source finder and updating model") # apply previous stefcal solutions to produce a corrected data image. # Make a restored image with 2000 CLEAN iterations. stefcal.stefcal(apply_only=True,output="CORR_DATA",plotvis=False, restore=dict(niter=2000),restore_lsm=False) ## now run pybdsm on the restored image, write output to a new LSM file lsm.pybdsm_search(threshold=5,output=LSM1) # the new sky model becomes the "current" LSM v.LSM = LSM1 # note that in principle this step is not needed (may as well go on directly # to the dE solutions, below), but it is instructive, as it will produce an intermediate-stage # image. But to save time, set restore=False to skip making clean images info("########## recalibrating with new model") stefcal.stefcal(stefcal_reset_all=True,output="CORR_RES",restore=False,flag_threshold=(1,.5)); # if we have a "reference" sky model configured, use it to set dE tags on # sources in our new sky model. Sources with dE tags will have DDE solutions. # The cal.transfer_tags() function sets the specified tag on any source # near enough to a reference model source with that tag. if LSMREF: lsm.transfer_tags(LSMREF,LSM,tags="dE",tolerance=45*ARCSEC) info("########## recalibrating with dEs") # re-run stefcal with the new model (and with direction dependent # solutions on dE-tagged sources, if any). Make a corrected residual # image, run CLEAN on it, restore the LSM sources back into the # image stefcal.stefcal(stefcal_reset_all=True,diffgains=True, output="CORR_RES",restore=dict(niter=1000), restore_lsm=True,flag_threshold=(1,.5)) # no reference LSM? then still need to make a restored image from the step above else: imager.make_image(dirty=False,restore=dict(niter=1000),restore_lsm=True); # the resulting image filename is in cal.FULLREST_IMAGE, # add it to our image list file IMAGE_LIST.add("${imager.FULLREST_IMAGE}");
def c_cal(goto_step=1): """Calibration for C-config data""" ## initial calibration if goto_step > 1: info("########## restarting calibration from step $goto_step") # Calibration step -- this is just a user-defined label used in filenames (etc. "blahblah_s1"), which serves to keep the output from each step # of a pipeline separate. If this is numeric, then functions such as stefcal.stefcal() will increment it automatically first thing. Otherwise you can set # it yourself to some more fancy label. Here we also provide a way to hop to particular step via goto_step v.STEP = goto_step - 1 if goto_step < 2: # set the superglobal LSM v.LSM = LSM0 info("########## solving for G with initial LSM") # no w-proj for dirty map to save time stefcal.stefcal(stefcal_reset_all=True, dirty=dict(wprojplanes=0, npix=NPIX), restore=dict(npix=NPIX, threshold=CLEAN_THRESH[0], wprojplanes=128)) info("########## running source finder and updating model") ## now run pybdsm on restored image, output LSM will be given by variable cal.PYBDSM_OUTPUT ### NB: select on radius to exclude any artefacts picked up around 3C147 itself lsm.pybdsm_search(thresh_pix=THRESH_PIX, thresh_isl=THRESH_ISL, select="r.gt.30s") ### merge new sources into sky model, give it a new name ($LSM1) lsm.tigger_convert("$LSM -a ${lsm.PYBDSM_OUTPUT} $LSM1 --rename -f") if goto_step < 3: info("########## solving for G+dE with updated LSM (initial+pybdsm)") v.LSM = LSM1 # now, set dE tags on sources lsm.transfer_tags(LSMREF, LSM, tags="dE", tolerance=45 * ARCSEC) stefcal.stefcal(stefcal_reset_all=True, diffgains=True, dirty=dict(wprojplanes=0, npix=NPIX)) if goto_step < 4: info("########## re-solving for G to apply IFR solutions") v.LSM = LSM1 stefcal.stefcal(diffgains=True, diffgain_apply_only=True, dirty=dict(wprojplanes=0, npix=NPIX), restore=dict(npix=NPIX, threshold=CLEAN_THRESH[1])) info("########## adding clean components to LSM") ff = pyfits.open(imager.MODEL_IMAGE) dd = ff[0].data dd *= 1.0769 # scale up to compensate for selfcal flux suppression # dd[dd<0] = 0; # remove negative components ff.writeto(LSM_CCMODEL, clobber=True) # add model image to LSM lsm.tigger_convert("$LSM $LSM2 --add-brick=ccmodel:$LSM_CCMODEL:2 -f") if goto_step < 5: info("########## re-running DD solutions") v.LSM = LSM2 # make final image stefcal.stefcal(dirty=dict(wprojplanes=0, npix=NPIX), diffgains=True, restore=dict(npix=NPIX, threshold=CLEAN_THRESH[2]), label="dE") # make per-channel cube makecube(NPIX) # make noise images makenoise()
def jointcal (goto_step=1,last_step=10,lsmbase=None,STEPS=None): """Calibration for joint C and D-config data""" info(">>>>>>>>>>>>> output directory is $OUTDIR. Please set OUTDIR explicitly to override"); # setup LSM filenames based on the full MS # note that these get interpolated once and for all here (and the _Template definitions above # get cancelled due to the explicit assignment here). The reason for doing it like this # is because I don't want these names to be changing due to the templates every time the # MS changes in a per(MS) call. v.FULLMS = MS LSM1 = II("$DESTDIR/$LSMBASE$SUFFIX+pybdsm.lsm.html"); LSM2 = II("$DESTDIR/$LSMBASE$SUFFIX+pybdsm2.lsm.html"); LSM3 = II("$DESTDIR/$LSMBASE$SUFFIX+pybdsm2+cc.lsm.html"); LSM_CCMODEL = II("$DESTDIR/$LSMBASE$SUFFIX+ccmodel.fits"); saveconf() stefcal.STEFCAL_DIFFGAIN_SMOOTHING = DE_SMOOTHING if not DE_INTERVALS else None; stefcal.STEFCAL_DIFFGAIN_INTERVALS = DE_INTERVALS; if DE_INTERVALS and TILE: stefcal.STEFCAL_TDLOPTS = "ms_sel.tile_size=%d"%(TILE*DE_INTERVALS[0]); # make MS list from sub-MSs import glob v.MS_List = glob.glob(MS+"/SUBMSS/*MS"); info("MS list is $MS_List"); if not MS_List: abort("No sub-MSs found"); imager.npix = NPIX = 4096 imager.cellsize = "2arcsec" imager.wprojplanes = 128 imager.CLEAN_ALGORITHM = "csclean" v.LSMREF = "${MS:BASE}.refmodel.lsm.html" THRESH_PIX,THRESH_ISL = (50,10),(15,5) CLEAN_THRESH = ".4mJy",".1mJy",".05mJy" stefcal.STEFCAL_STEP_INCR = 0 # precvent stefcal from auto-incrementing v.STEP: we set the step counter explicitly here if STEPS is None: STEPS = list(numpy.arange(goto_step,last_step+.1,.5)); STEPS = map(float,STEPS); if STEPS[0] != 1: info("########## restarting calibration from step %.1f"%STEPS[0]); if lsmbase: LSMBASE = lsmbase; ## initial calibration if 1. in STEPS: info("########## step 1: solving for G with initial LSM"); v.LSM,v.STEP = LSM0,1 per_ms(jointcal_g); if 1.5 in STEPS: info("########## step 1.5: making joint image"); v.LSM,v.STEP = LSM0,1 v.MS = FULLMS # initial model is total flux only, made from a 2x size image to catch distant sources imager.make_image(dirty=False,stokes="I",restore=dict(npix=NPIX*2,threshold=CLEAN_THRESH[0],wprojplanes=128),restore_lsm=False); info("########## running source finder and updating model"); ## now run pybdsm on restored image, output LSM will be given by variable cal.PYBDSM_OUTPUT ### NB: select on radius to exclude any artefacts picked up around 3C147 itself lsm.pybdsm_search(thresh_pix=THRESH_PIX[0],thresh_isl=THRESH_ISL[0],select="r.gt.30s",pol=False); ### merge new sources into sky model, give it a new name ($LSM1) lsm.tigger_convert("$LSM -a ${lsm.PYBDSM_OUTPUT} $LSM1 --rename -f"); # if 2. in STEPS: # info("########## step 2: repeating G solution"); # v.LSM,v.STEP = LSM1,2 # v.MS = FULLMS # per_ms(jointcal_g); if 2. in STEPS: info("########## step 2: initial dE solution"); v.LSM,v.STEP = LSM1,2 v.MS = FULLMS # now, set dE tags on sources lsm.transfer_tags(LSMREF,LSM,tags="dE",tolerance=45*ARCSEC); per_ms(jointcal_de_reset); if 3. in STEPS: info("########## step 3: re-solving for G to apply IFR solutions"); v.LSM,v.STEP = LSM1,3 v.MS = FULLMS per_ms(jointcal_de_apply); info("########## running source finder and updating model"); v.MS = FULLMS imager.make_image(dirty=False,stokes="IV",restore=dict(npix=NPIX,threshold=CLEAN_THRESH[1],wprojplanes=128),restore_lsm=False); ## now run pybdsm on restored image, output LSM will be given by variable cal.PYBDSM_OUTPUT ### NB: select on radius to exclude any artefacts picked up around 3C147 itself lsm.pybdsm_search(thresh_pix=THRESH_PIX[1],thresh_isl=THRESH_ISL[1],select="r.gt.30s"); ### merge new sources into sky model, give it a new name ($LSM1) lsm.tigger_convert("$LSM -a ${lsm.PYBDSM_OUTPUT} $LSM2 --rename -f"); if 4. in STEPS: info("########## step 4: solving for G+dE with updated LSM (initial+pybdsm^2)"); v.MS = FULLMS v.LSM,v.STEP = LSM2,4 lsm.transfer_tags(LSMREF,LSM,tags="dE",tolerance=45*ARCSEC); per_ms(jointcal_de); v.MS = FULLMS imager.make_image(dirty=False,stokes="IV",restore=dict(npix=NPIX,threshold=CLEAN_THRESH[1],wprojplanes=128),restore_lsm=False); info("########## adding clean components to LSM"); ff = pyfits.open(imager.MODEL_IMAGE); dd = ff[0].data; dd *= 1.0769 # scale up to compensate for selfcal flux suppression # dd[dd<0] = 0; # remove negative components ff.writeto(LSM_CCMODEL,clobber=True); # add model image to LSM lsm.tigger_convert("$LSM $LSM3 --add-brick=ccmodel:$LSM_CCMODEL:2 -f"); if 5. in STEPS: info("########## step 5: re-running DD solutions"); v.MS = FULLMS v.LSM,v.STEP = LSM3,5 per_ms(jointcal_de_final); if 5.5 in STEPS: info("########## step 5.5: making joint image"); v.MS = FULLMS v.LSM,v.STEP = LSM3,5 imager.make_image(dirty=False,stokes="IQUV",restore=dict(npix=NPIX,threshold=CLEAN_THRESH[2],wprojplanes=128),restore_lsm=True); if 6. in STEPS: info("########## step 6: noise sim"); per_ms(lambda:makecube(stokes="IQUV")); v.LSM,v.STEP = LSM3,5 v.MS = FULLMS; makecube(stokes="IQUV"); makenoise();
def jointcal(goto_step=1, last_step=10, lsmbase=None, STEPS=None): """Calibration for joint C and D-config data""" info( ">>>>>>>>>>>>> output directory is $OUTDIR. Please set OUTDIR explicitly to override" ) # setup LSM filenames based on the full MS # note that these get interpolated once and for all here (and the _Template definitions above # get cancelled due to the explicit assignment here). The reason for doing it like this # is because I don't want these names to be changing due to the templates every time the # MS changes in a per(MS) call. v.FULLMS = MS LSM1 = II("$DESTDIR/$LSMBASE$SUFFIX+pybdsm.lsm.html") LSM2 = II("$DESTDIR/$LSMBASE$SUFFIX+pybdsm2.lsm.html") LSM3 = II("$DESTDIR/$LSMBASE$SUFFIX+pybdsm2+cc.lsm.html") LSM_CCMODEL = II("$DESTDIR/$LSMBASE$SUFFIX+ccmodel.fits") saveconf() stefcal.STEFCAL_DIFFGAIN_SMOOTHING = DE_SMOOTHING if not DE_INTERVALS else None stefcal.STEFCAL_DIFFGAIN_INTERVALS = DE_INTERVALS if DE_INTERVALS and TILE: stefcal.STEFCAL_TDLOPTS = "ms_sel.tile_size=%d" % (TILE * DE_INTERVALS[0]) # make MS list from sub-MSs import glob v.MS_List = glob.glob(MS + "/SUBMSS/*MS") info("MS list is $MS_List") if not MS_List: abort("No sub-MSs found") imager.npix = NPIX = 4096 imager.cellsize = "2arcsec" imager.wprojplanes = 128 imager.CLEAN_ALGORITHM = "csclean" v.LSMREF = "${MS:BASE}.refmodel.lsm.html" THRESH_PIX, THRESH_ISL = (50, 10), (15, 5) CLEAN_THRESH = ".4mJy", ".1mJy", ".05mJy" stefcal.STEFCAL_STEP_INCR = 0 # precvent stefcal from auto-incrementing v.STEP: we set the step counter explicitly here if STEPS is None: STEPS = list(numpy.arange(goto_step, last_step + .1, .5)) STEPS = map(float, STEPS) if STEPS[0] != 1: info("########## restarting calibration from step %.1f" % STEPS[0]) if lsmbase: LSMBASE = lsmbase ## initial calibration if 1. in STEPS: info("########## step 1: solving for G with initial LSM") v.LSM, v.STEP = LSM0, 1 per_ms(jointcal_g) if 1.5 in STEPS: info("########## step 1.5: making joint image") v.LSM, v.STEP = LSM0, 1 v.MS = FULLMS # initial model is total flux only, made from a 2x size image to catch distant sources imager.make_image(dirty=False, stokes="I", restore=dict(npix=NPIX * 2, threshold=CLEAN_THRESH[0], wprojplanes=128), restore_lsm=False) info("########## running source finder and updating model") ## now run pybdsm on restored image, output LSM will be given by variable cal.PYBDSM_OUTPUT ### NB: select on radius to exclude any artefacts picked up around 3C147 itself lsm.pybdsm_search(thresh_pix=THRESH_PIX[0], thresh_isl=THRESH_ISL[0], select="r.gt.30s", pol=False) ### merge new sources into sky model, give it a new name ($LSM1) lsm.tigger_convert("$LSM -a ${lsm.PYBDSM_OUTPUT} $LSM1 --rename -f") # if 2. in STEPS: # info("########## step 2: repeating G solution"); # v.LSM,v.STEP = LSM1,2 # v.MS = FULLMS # per_ms(jointcal_g); if 2. in STEPS: info("########## step 2: initial dE solution") v.LSM, v.STEP = LSM1, 2 v.MS = FULLMS # now, set dE tags on sources lsm.transfer_tags(LSMREF, LSM, tags="dE", tolerance=45 * ARCSEC) per_ms(jointcal_de_reset) if 3. in STEPS: info("########## step 3: re-solving for G to apply IFR solutions") v.LSM, v.STEP = LSM1, 3 v.MS = FULLMS per_ms(jointcal_de_apply) info("########## running source finder and updating model") v.MS = FULLMS imager.make_image(dirty=False, stokes="IV", restore=dict(npix=NPIX, threshold=CLEAN_THRESH[1], wprojplanes=128), restore_lsm=False) ## now run pybdsm on restored image, output LSM will be given by variable cal.PYBDSM_OUTPUT ### NB: select on radius to exclude any artefacts picked up around 3C147 itself lsm.pybdsm_search(thresh_pix=THRESH_PIX[1], thresh_isl=THRESH_ISL[1], select="r.gt.30s") ### merge new sources into sky model, give it a new name ($LSM1) lsm.tigger_convert("$LSM -a ${lsm.PYBDSM_OUTPUT} $LSM2 --rename -f") if 4. in STEPS: info( "########## step 4: solving for G+dE with updated LSM (initial+pybdsm^2)" ) v.MS = FULLMS v.LSM, v.STEP = LSM2, 4 lsm.transfer_tags(LSMREF, LSM, tags="dE", tolerance=45 * ARCSEC) per_ms(jointcal_de) v.MS = FULLMS imager.make_image(dirty=False, stokes="IV", restore=dict(npix=NPIX, threshold=CLEAN_THRESH[1], wprojplanes=128), restore_lsm=False) info("########## adding clean components to LSM") ff = pyfits.open(imager.MODEL_IMAGE) dd = ff[0].data dd *= 1.0769 # scale up to compensate for selfcal flux suppression # dd[dd<0] = 0; # remove negative components ff.writeto(LSM_CCMODEL, clobber=True) # add model image to LSM lsm.tigger_convert("$LSM $LSM3 --add-brick=ccmodel:$LSM_CCMODEL:2 -f") if 5. in STEPS: info("########## step 5: re-running DD solutions") v.MS = FULLMS v.LSM, v.STEP = LSM3, 5 per_ms(jointcal_de_final) if 5.5 in STEPS: info("########## step 5.5: making joint image") v.MS = FULLMS v.LSM, v.STEP = LSM3, 5 imager.make_image(dirty=False, stokes="IQUV", restore=dict(npix=NPIX, threshold=CLEAN_THRESH[2], wprojplanes=128), restore_lsm=True) if 6. in STEPS: info("########## step 6: noise sim") per_ms(lambda: makecube(stokes="IQUV")) v.LSM, v.STEP = LSM3, 5 v.MS = FULLMS makecube(stokes="IQUV") makenoise()
def selfcal_loop_pipeline(goto_step=0.): if not os.path.isdir(v.DESTDIR): os.makedirs(v.DESTDIR) #copy data to corrected data column to run clean if goto_step <= 0.: v.STEP=1 info("########## adding columns %i"%v.STEP) for msFile in sorted(v.MS_List):pyrap.tables.addImagingColumns(msFile) if goto_step <= 0.5: v.STEP=1 info("########## copying data to corrected data column step %i"%v.STEP) pper("MS",copy_data_to_corrected_data) #run a basic clean to perform source finding on if goto_step <= 1: v.STEP=1 info("########## making initial clean image step %i"%v.STEP) wscleanDict={ 'name':'wsclean', 'datacolumn':'CORRECTED_DATA', 'size': '1024 1024', #'scale': 330./3600., 'scale': 0.11, 'niter': 250, 'threshold': 1., 'pol': 'I', #'weight': 'natural', 'weight': 'briggs 0.0', 'nwlayers':32, 'channelrange': '%i %i'%(startChan,endChan), 'channelsout': 1} for msFile in sorted(v.MS_List): msBase=msFile.split('/')[-1].split('.MS')[0] wscleanDict['name']=msBase+'_s%i'%v.STEP run_wsclean(msFile,wscleanDict) fitsfiles=glob.glob('*.fits') for ff in fitsfiles: shutil.move(ff,v.DESTDIR) #derive an initial sky model from the clean'd image if goto_step <= 1.5: v.STEP=1 fitsfiles=glob.glob(v.DESTDIR+'/*_s1-image.fits') for ff in sorted(fitsfiles): olsm=ff.split('-image.fits')[0]+'.lsm' lsm.pybdsm_search(image=ff,output=olsm,thresh_pix=20,thresh_isl=15) lsm.tigger_convert(olsm) #run stefcal using the initial sky model if goto_step <= 2: v.STEP=2 lsm0files=glob.glob(v.DESTDIR+'/*_s1.lsm.html') wscleanDict={ 'name':'wsclean', 'datacolumn':'CORRECTED_DATA', 'size': '1024 1024', #'scale': 330./3600., 'scale': 0.11, 'niter': 500, 'threshold': 1., 'pol': 'I', #'weight': 'natural', 'weight': 'briggs 0.0', 'nwlayers':32, 'channelrange': '%i %i'%(startChan,endChan), 'channelsout': 1} for msFile in sorted(v.MS_List): msBase=msFile.split('/')[-1].split('.MS')[0] lsm0=filter(lambda x: x.split('/')[-1].startswith(msBase),lsm0files)[0] v.LSM=lsm0 v.MS=msFile stefcal.STEFCAL_STEP_INCR = 0 # we set the step counter explicitly info("########## initial calibration of %s using sky model %s step %i"%(msFile,lsm0,v.STEP)) stefcal.stefcal(output='CORR_DATA',dirty=False,restore=False,section='calico-stefcal') info("########## making clean image step %i"%v.STEP) wscleanDict['name']=msBase+'_s%i'%v.STEP run_wsclean(msFile,wscleanDict) fitsfiles=glob.glob('*.fits') for ff in fitsfiles: shutil.move(ff,v.DESTDIR) #derive an second sky model from the clean'd image if goto_step <= 2.5: v.STEP=2 fitsfiles=glob.glob(v.DESTDIR+'/*_s2-image.fits') for ff in sorted(fitsfiles): olsm=ff.split('-image.fits')[0]+'.lsm' lsm.pybdsm_search(image=ff,output=olsm,thresh_pix=15,thresh_isl=10) lsm.tigger_convert(olsm) #run stefcal using the second sky model if goto_step <= 3: v.STEP=3 lsm1files=glob.glob(v.DESTDIR+'/*_s2.lsm.html') wscleanDict={ 'name':'wsclean', 'datacolumn':'CORRECTED_DATA', 'size': '1024 1024', #'scale': 330./3600., 'scale': 0.11, 'niter': 1000, 'threshold': 1., 'pol': 'I', #'weight': 'natural', 'weight': 'briggs 0.0', 'nwlayers':32, 'channelrange': '%i %i'%(startChan,endChan), 'channelsout': 1} for msFile in sorted(v.MS_List): x.sh('export LC_NUMERIC=en_GB.utf8') msBase=msFile.split('/')[-1].split('.MS')[0] lsm1=filter(lambda x: x.split('/')[-1].startswith(msBase),lsm1files)[0] v.LSM=lsm1 v.MS=msFile stefcal.STEFCAL_STEP_INCR = 0 # we set the step counter explicitly info("########## second calibration of %s using sky model %s step %i"%(msFile,lsm1,v.STEP)) stefcal.stefcal(output='CORR_DATA',dirty=False,restore=False,section='calico-stefcal') info("########## making clean image step %i"%v.STEP) wscleanDict['name']=msBase+'_s%i'%v.STEP run_wsclean(msFile,wscleanDict) fitsfiles=glob.glob('*.fits') for ff in fitsfiles: shutil.move(ff,v.DESTDIR) #derive an sky model from the clean'd image if goto_step <= 3.5: v.STEP=3 fitsfiles=glob.glob(v.DESTDIR+'/*_s3-image.fits') for ff in sorted(fitsfiles): olsm=ff.split('-image.fits')[0]+'.lsm' lsm.pybdsm_search(image=ff,output=olsm,thresh_pix=12,thresh_isl=5) lsm.tigger_convert(olsm) #run stefcal using the third sky model and produce final complex, multi-frequency images if goto_step <= 4: v.STEP=4 lsm2files=glob.glob(v.DESTDIR+'/*_s3.lsm.html') wscleanDict={ 'name':'wsclean', 'datacolumn':'CORRECTED_DATA', 'size': '1024 1024', #'scale': 330./3600., 'scale': 0.11, 'niter': 1000, 'threshold': 1., 'pol': 'xx,xy,yx,yy', #'weight': 'natural', 'weight': 'briggs 0.0', 'nwlayers':32, 'channelrange': '%i %i'%(startChan,endChan), 'channelsout': nsubbands, 'joinpolarizations': None} for msFile in sorted(v.MS_List): x.sh('export LC_NUMERIC=en_GB.utf8') msBase=msFile.split('/')[-1].split('.MS')[0] lsm2=filter(lambda x: x.split('/')[-1].startswith(msBase),lsm2files)[0] v.LSM=lsm2 v.MS=msFile stefcal.STEFCAL_STEP_INCR = 0 # we set the step counter explicitly info("########## second calibration of %s using sky model %s step %i"%(msFile,lsm2,v.STEP)) stefcal.stefcal(output='CORR_DATA',dirty=False,restore=False,section='calico-stefcal') wscleanDict['name']=msBase+'_s%i'%v.STEP run_wsclean(msFile,wscleanDict) fitsfiles=glob.glob('*.fits') for ff in fitsfiles: shutil.move(ff,v.DESTDIR) #clean up if goto_step <= 5: gaulfiles=glob.glob('*.gaul') for gf in gaulfiles: shutil.move(gf,v.DESTDIR)
def phase_precal_selfcal_pipeline(goto_step=3.): useStep = 1 skipCalMSlist = [ 'zen.2455819.25927.uvcRREM', 'zen.2455819.27319.uvcRREM', 'zen.2455819.32190.uvcRREM', 'zen.2455819.34278.uvcRREM', 'zen.2455819.34974.uvcRREM', 'zen.2455819.38454.uvcRREM', 'zen.2455819.40542.uvcRREM', 'zen.2455819.41238.uvcRREM', 'zen.2455819.41934.uvcRREM', 'zen.2455819.42630.uvcRREM', 'zen.2455819.73947.uvcRREM', 'zen.2455819.74643.uvcRREM', 'zen.2455819.75339.uvcRREM', 'zen.2455819.76035.uvcRREM', 'zen.2455819.78819.uvcRREM', 'zen.2455819.79515.uvcRREM', 'zen.2455819.80211.uvcRREM', 'zen.2455819.80906.uvcRREM' ] #MS files to skip calibration on skipCleanMSlist = ['zen.2455819.79515.uvcRREM' ] #problematic MS files when imaging multi-freq if not os.path.isdir(v.DESTDIR): os.makedirs(v.DESTDIR) if goto_step <= 0.: v.STEP = 1 info("########## adding columns %i" % v.STEP) for msFile in sorted(v.MS_List): pyrap.tables.addImagingColumns(msFile) #copy data to corrected data column to run clean if goto_step <= 0.5: v.STEP = 1 info("########## copying data to corrected data column step %i" % v.STEP) pper("MS", copy_data_to_corrected_data) #run a basic clean to perform source finding on if goto_step <= 1: v.STEP = 1 info("########## making initial clean image step %i" % v.STEP) wscleanDict = { 'name': 'wsclean', 'datacolumn': 'CORRECTED_DATA', 'size': '1024 1024', #'scale': 330./3600., 'scale': 0.11, 'niter': 250, 'threshold': 100., 'pol': 'I', #'weight': 'natural', 'weight': 'briggs 0.0', 'nwlayers': 32, 'channelrange': '%i %i' % (startChan, endChan), 'channelsout': 1 } for msFile in sorted(v.MS_List): msBase = msFile.split('/')[-1].split('.MS')[0] wscleanDict['name'] = msBase + '_s%i' % v.STEP run_wsclean(msFile, wscleanDict) fitsfiles = glob.glob('*.fits') for ff in fitsfiles: shutil.move(ff, v.DESTDIR) #derive a shallow depth sky model from the clean'd image if goto_step <= 1.5: v.STEP = 1 fitsfiles = glob.glob(v.DESTDIR + '/*_s1-image.fits') for ff in sorted(fitsfiles): olsm = ff.split('-image.fits')[0] + '.lsm' lsm.pybdsm_search(image=ff, output=olsm, thresh_pix=20, thresh_isl=15) lsm.tigger_convert(olsm) #run stefcal using the sky model, produce a Stokes I image for source finding if goto_step <= 2: v.STEP = 2 lsm0files = glob.glob(v.DESTDIR + '/*_s1.lsm.html') wscleanDict = { 'name': 'wsclean', 'datacolumn': 'CORRECTED_DATA', 'size': '1024 1024', #'scale': 330./3600., 'scale': 0.11, 'niter': 250, 'threshold': 100., 'pol': 'I', #'weight': 'natural', 'weight': 'briggs 0.0', 'nwlayers': 32, 'channelrange': '%i %i' % (startChan, endChan), 'channelsout': 1 } for msFile in sorted(v.MS_List): msBase = msFile.split('/')[-1].split('.MS')[0] if msBase in skipCalMSlist: info('Skipping this MS file') else: x.sh('export LC_NUMERIC=en_GB.utf8') lsm0 = filter(lambda x: x.split('/')[-1].startswith(msBase), lsm0files)[0] v.LSM = lsm0 v.MS = msFile stefcal.STEFCAL_STEP_INCR = 0 # we set the step counter explicitly info( "########## initial calibration of %s using sky model %s step %i" % (msFile, lsm0, v.STEP)) stefcal.stefcal(output='CORR_DATA', dirty=False, restore=False, section='calico-stefcal') info("########## making clean image step %i" % v.STEP) wscleanDict['name'] = msBase + '_s%i' % v.STEP run_wsclean(msFile, wscleanDict) fitsfiles = glob.glob('*.fits') for ff in fitsfiles: shutil.move(ff, v.DESTDIR) #derive a deeper depth sky model from the clean'd image if goto_step <= 2.5: v.STEP = 2 fitsfiles = glob.glob(v.DESTDIR + '/*_s2-image.fits') for ff in sorted(fitsfiles): olsm = ff.split('-image.fits')[0] + '.lsm' lsm.pybdsm_search(image=ff, output=olsm, thresh_pix=10, thresh_isl=7) lsm.tigger_convert(olsm) #generate multi-frequency, complex images for image domain corrections if goto_step <= 3: v.STEP = 3 wscleanDict = { 'name': 'wsclean', 'datacolumn': 'CORRECTED_DATA', 'size': '1024 1024', #'scale': 330./3600., 'scale': 0.11, #'niter': 1000, 'niter': 1, 'threshold': 100., 'pol': 'xx,xy,yx,yy', 'weight': 'natural', #'weight': 'briggs 0.0', 'nwlayers': 32, 'channelrange': '%i %i' % (startChan, endChan), 'channelsout': nsubbands, 'joinpolarizations': None } for msFile in sorted(v.MS_List): x.sh('export LC_NUMERIC=en_GB.utf8') msBase = msFile.split('/')[-1].split('.MS')[0] if msBase in skipCleanMSlist: wscleanDict['niter'] = 1 else: wscleanDict['niter'] = 1 #else: wscleanDict['niter']=1000 info("########## making clean image step %i" % v.STEP) wscleanDict['name'] = msBase + '_s%i' % v.STEP run_wsclean(msFile, wscleanDict) fitsfiles = glob.glob('*.fits') for ff in fitsfiles: shutil.move(ff, v.DESTDIR)
def cal_ms(): """cal_ms: runs a calibration loop over the current MS""" # initialize the calibration "step" counter and "label". These are used # to auto-generate output filenames; cal.stefcal() below automatically increments # cal.STEP v.STEP = 0 # set the superglobal LSM variable. See explanation for "v." in the text v.LSM = LSM0 # info(), warn() and abort() are Pyxis functions for writing output to the log info("########## solving for G with initial LSM") # do one stefcal run with the initial model, produce corrected residuals, # do simple flagging on the residuals. See cal.stefcal() for full docs. stefcal.stefcal(stefcal_reset_all=True, output="CORR_RES", restore=False, flag_threshold=(1, .5)) # If we got to this point, then assume things are rolling along fine. # Copy the recipe and config to the destination directory for future reference # (very useful when modifying recipes, so you can keep track of what settings # are responsible for what output!) xo.sh("cp pyxis*.py pyxis*.conf ${mqt.TDLCONFIG} $DESTDIR", shell=True) info( "########## remaking data image, running source finder and updating model" ) # apply previous stefcal solutions to produce a corrected data image. # Make a restored image with 2000 CLEAN iterations. stefcal.stefcal(apply_only=True, output="CORR_DATA", plotvis=False, restore=dict(niter=2000), restore_lsm=False) ## now run pybdsm on the restored image, write output to a new LSM file lsm.pybdsm_search(threshold=5, output=LSM1) # the new sky model becomes the "current" LSM v.LSM = LSM1 # note that in principle this step is not needed (may as well go on directly # to the dE solutions, below), but it is instructive, as it will produce an intermediate-stage # image. But to save time, set restore=False to skip making clean images info("########## recalibrating with new model") stefcal.stefcal(stefcal_reset_all=True, output="CORR_RES", restore=False, flag_threshold=(1, .5)) # if we have a "reference" sky model configured, use it to set dE tags on # sources in our new sky model. Sources with dE tags will have DDE solutions. # The cal.transfer_tags() function sets the specified tag on any source # near enough to a reference model source with that tag. if LSMREF: lsm.transfer_tags(LSMREF, LSM, tags="dE", tolerance=45 * ARCSEC) info("########## recalibrating with dEs") # re-run stefcal with the new model (and with direction dependent # solutions on dE-tagged sources, if any). Make a corrected residual # image, run CLEAN on it, restore the LSM sources back into the # image stefcal.stefcal(stefcal_reset_all=True, diffgains=True, output="CORR_RES", restore=dict(niter=1000), restore_lsm=True, flag_threshold=(1, .5)) # no reference LSM? then still need to make a restored image from the step above else: imager.make_image(dirty=False, restore=dict(niter=1000), restore_lsm=True) # the resulting image filename is in cal.FULLREST_IMAGE, # add it to our image list file IMAGE_LIST.add("${imager.FULLREST_IMAGE}")
def phase_precal_selfcal_pipeline(goto_step=3.): useStep=1 skipCalMSlist=[ 'zen.2455819.25927.uvcRREM', 'zen.2455819.27319.uvcRREM', 'zen.2455819.32190.uvcRREM', 'zen.2455819.34278.uvcRREM', 'zen.2455819.34974.uvcRREM', 'zen.2455819.38454.uvcRREM', 'zen.2455819.40542.uvcRREM', 'zen.2455819.41238.uvcRREM', 'zen.2455819.41934.uvcRREM', 'zen.2455819.42630.uvcRREM', 'zen.2455819.73947.uvcRREM', 'zen.2455819.74643.uvcRREM', 'zen.2455819.75339.uvcRREM', 'zen.2455819.76035.uvcRREM', 'zen.2455819.78819.uvcRREM', 'zen.2455819.79515.uvcRREM', 'zen.2455819.80211.uvcRREM', 'zen.2455819.80906.uvcRREM'] #MS files to skip calibration on skipCleanMSlist=['zen.2455819.79515.uvcRREM'] #problematic MS files when imaging multi-freq if not os.path.isdir(v.DESTDIR): os.makedirs(v.DESTDIR) if goto_step <= 0.: v.STEP=1 info("########## adding columns %i"%v.STEP) for msFile in sorted(v.MS_List):pyrap.tables.addImagingColumns(msFile) #copy data to corrected data column to run clean if goto_step <= 0.5: v.STEP=1 info("########## copying data to corrected data column step %i"%v.STEP) pper("MS",copy_data_to_corrected_data) #run a basic clean to perform source finding on if goto_step <= 1: v.STEP=1 info("########## making initial clean image step %i"%v.STEP) wscleanDict={ 'name':'wsclean', 'datacolumn':'CORRECTED_DATA', 'size': '1024 1024', #'scale': 330./3600., 'scale': 0.11, 'niter': 250, 'threshold': 100., 'pol': 'I', #'weight': 'natural', 'weight': 'briggs 0.0', 'nwlayers':32, 'channelrange': '%i %i'%(startChan,endChan), 'channelsout': 1} for msFile in sorted(v.MS_List): msBase=msFile.split('/')[-1].split('.MS')[0] wscleanDict['name']=msBase+'_s%i'%v.STEP run_wsclean(msFile,wscleanDict) fitsfiles=glob.glob('*.fits') for ff in fitsfiles: shutil.move(ff,v.DESTDIR) #derive a shallow depth sky model from the clean'd image if goto_step <= 1.5: v.STEP=1 fitsfiles=glob.glob(v.DESTDIR+'/*_s1-image.fits') for ff in sorted(fitsfiles): olsm=ff.split('-image.fits')[0]+'.lsm' lsm.pybdsm_search(image=ff,output=olsm,thresh_pix=20,thresh_isl=15) lsm.tigger_convert(olsm) #run stefcal using the sky model, produce a Stokes I image for source finding if goto_step <= 2: v.STEP=2 lsm0files=glob.glob(v.DESTDIR+'/*_s1.lsm.html') wscleanDict={ 'name':'wsclean', 'datacolumn':'CORRECTED_DATA', 'size': '1024 1024', #'scale': 330./3600., 'scale': 0.11, 'niter': 250, 'threshold': 100., 'pol': 'I', #'weight': 'natural', 'weight': 'briggs 0.0', 'nwlayers':32, 'channelrange': '%i %i'%(startChan,endChan), 'channelsout': 1} for msFile in sorted(v.MS_List): msBase=msFile.split('/')[-1].split('.MS')[0] if msBase in skipCalMSlist: info('Skipping this MS file') else: x.sh('export LC_NUMERIC=en_GB.utf8') lsm0=filter(lambda x: x.split('/')[-1].startswith(msBase),lsm0files)[0] v.LSM=lsm0 v.MS=msFile stefcal.STEFCAL_STEP_INCR = 0 # we set the step counter explicitly info("########## initial calibration of %s using sky model %s step %i"%(msFile,lsm0,v.STEP)) stefcal.stefcal(output='CORR_DATA',dirty=False,restore=False,section='calico-stefcal') info("########## making clean image step %i"%v.STEP) wscleanDict['name']=msBase+'_s%i'%v.STEP run_wsclean(msFile,wscleanDict) fitsfiles=glob.glob('*.fits') for ff in fitsfiles: shutil.move(ff,v.DESTDIR) #derive a deeper depth sky model from the clean'd image if goto_step <= 2.5: v.STEP=2 fitsfiles=glob.glob(v.DESTDIR+'/*_s2-image.fits') for ff in sorted(fitsfiles): olsm=ff.split('-image.fits')[0]+'.lsm' lsm.pybdsm_search(image=ff,output=olsm,thresh_pix=10,thresh_isl=7) lsm.tigger_convert(olsm) #generate multi-frequency, complex images for image domain corrections if goto_step <= 3: v.STEP=3 wscleanDict={ 'name':'wsclean', 'datacolumn':'CORRECTED_DATA', 'size': '1024 1024', #'scale': 330./3600., 'scale': 0.11, #'niter': 1000, 'niter': 1, 'threshold': 100., 'pol': 'xx,xy,yx,yy', 'weight': 'natural', #'weight': 'briggs 0.0', 'nwlayers':32, 'channelrange': '%i %i'%(startChan,endChan), 'channelsout': nsubbands, 'joinpolarizations': None} for msFile in sorted(v.MS_List): x.sh('export LC_NUMERIC=en_GB.utf8') msBase=msFile.split('/')[-1].split('.MS')[0] if msBase in skipCleanMSlist: wscleanDict['niter']=1 else: wscleanDict['niter']=1 #else: wscleanDict['niter']=1000 info("########## making clean image step %i"%v.STEP) wscleanDict['name']=msBase+'_s%i'%v.STEP run_wsclean(msFile,wscleanDict) fitsfiles=glob.glob('*.fits') for ff in fitsfiles: shutil.move(ff,v.DESTDIR)
def selfcal_loop_pipeline(goto_step=0.): if not os.path.isdir(v.DESTDIR): os.makedirs(v.DESTDIR) #copy data to corrected data column to run clean if goto_step <= 0.: v.STEP = 1 info("########## adding columns %i" % v.STEP) for msFile in sorted(v.MS_List): pyrap.tables.addImagingColumns(msFile) if goto_step <= 0.5: v.STEP = 1 info("########## copying data to corrected data column step %i" % v.STEP) pper("MS", copy_data_to_corrected_data) #run a basic clean to perform source finding on if goto_step <= 1: v.STEP = 1 info("########## making initial clean image step %i" % v.STEP) wscleanDict = { 'name': 'wsclean', 'datacolumn': 'CORRECTED_DATA', 'size': '1024 1024', #'scale': 330./3600., 'scale': 0.11, 'niter': 250, 'threshold': 1., 'pol': 'I', #'weight': 'natural', 'weight': 'briggs 0.0', 'nwlayers': 32, 'channelrange': '%i %i' % (startChan, endChan), 'channelsout': 1 } for msFile in sorted(v.MS_List): msBase = msFile.split('/')[-1].split('.MS')[0] wscleanDict['name'] = msBase + '_s%i' % v.STEP run_wsclean(msFile, wscleanDict) fitsfiles = glob.glob('*.fits') for ff in fitsfiles: shutil.move(ff, v.DESTDIR) #derive an initial sky model from the clean'd image if goto_step <= 1.5: v.STEP = 1 fitsfiles = glob.glob(v.DESTDIR + '/*_s1-image.fits') for ff in sorted(fitsfiles): olsm = ff.split('-image.fits')[0] + '.lsm' lsm.pybdsm_search(image=ff, output=olsm, thresh_pix=20, thresh_isl=15) lsm.tigger_convert(olsm) #run stefcal using the initial sky model if goto_step <= 2: v.STEP = 2 lsm0files = glob.glob(v.DESTDIR + '/*_s1.lsm.html') wscleanDict = { 'name': 'wsclean', 'datacolumn': 'CORRECTED_DATA', 'size': '1024 1024', #'scale': 330./3600., 'scale': 0.11, 'niter': 500, 'threshold': 1., 'pol': 'I', #'weight': 'natural', 'weight': 'briggs 0.0', 'nwlayers': 32, 'channelrange': '%i %i' % (startChan, endChan), 'channelsout': 1 } for msFile in sorted(v.MS_List): msBase = msFile.split('/')[-1].split('.MS')[0] lsm0 = filter(lambda x: x.split('/')[-1].startswith(msBase), lsm0files)[0] v.LSM = lsm0 v.MS = msFile stefcal.STEFCAL_STEP_INCR = 0 # we set the step counter explicitly info( "########## initial calibration of %s using sky model %s step %i" % (msFile, lsm0, v.STEP)) stefcal.stefcal(output='CORR_DATA', dirty=False, restore=False, section='calico-stefcal') info("########## making clean image step %i" % v.STEP) wscleanDict['name'] = msBase + '_s%i' % v.STEP run_wsclean(msFile, wscleanDict) fitsfiles = glob.glob('*.fits') for ff in fitsfiles: shutil.move(ff, v.DESTDIR) #derive an second sky model from the clean'd image if goto_step <= 2.5: v.STEP = 2 fitsfiles = glob.glob(v.DESTDIR + '/*_s2-image.fits') for ff in sorted(fitsfiles): olsm = ff.split('-image.fits')[0] + '.lsm' lsm.pybdsm_search(image=ff, output=olsm, thresh_pix=15, thresh_isl=10) lsm.tigger_convert(olsm) #run stefcal using the second sky model if goto_step <= 3: v.STEP = 3 lsm1files = glob.glob(v.DESTDIR + '/*_s2.lsm.html') wscleanDict = { 'name': 'wsclean', 'datacolumn': 'CORRECTED_DATA', 'size': '1024 1024', #'scale': 330./3600., 'scale': 0.11, 'niter': 1000, 'threshold': 1., 'pol': 'I', #'weight': 'natural', 'weight': 'briggs 0.0', 'nwlayers': 32, 'channelrange': '%i %i' % (startChan, endChan), 'channelsout': 1 } for msFile in sorted(v.MS_List): x.sh('export LC_NUMERIC=en_GB.utf8') msBase = msFile.split('/')[-1].split('.MS')[0] lsm1 = filter(lambda x: x.split('/')[-1].startswith(msBase), lsm1files)[0] v.LSM = lsm1 v.MS = msFile stefcal.STEFCAL_STEP_INCR = 0 # we set the step counter explicitly info( "########## second calibration of %s using sky model %s step %i" % (msFile, lsm1, v.STEP)) stefcal.stefcal(output='CORR_DATA', dirty=False, restore=False, section='calico-stefcal') info("########## making clean image step %i" % v.STEP) wscleanDict['name'] = msBase + '_s%i' % v.STEP run_wsclean(msFile, wscleanDict) fitsfiles = glob.glob('*.fits') for ff in fitsfiles: shutil.move(ff, v.DESTDIR) #derive an sky model from the clean'd image if goto_step <= 3.5: v.STEP = 3 fitsfiles = glob.glob(v.DESTDIR + '/*_s3-image.fits') for ff in sorted(fitsfiles): olsm = ff.split('-image.fits')[0] + '.lsm' lsm.pybdsm_search(image=ff, output=olsm, thresh_pix=12, thresh_isl=5) lsm.tigger_convert(olsm) #run stefcal using the third sky model and produce final complex, multi-frequency images if goto_step <= 4: v.STEP = 4 lsm2files = glob.glob(v.DESTDIR + '/*_s3.lsm.html') wscleanDict = { 'name': 'wsclean', 'datacolumn': 'CORRECTED_DATA', 'size': '1024 1024', #'scale': 330./3600., 'scale': 0.11, 'niter': 1000, 'threshold': 1., 'pol': 'xx,xy,yx,yy', #'weight': 'natural', 'weight': 'briggs 0.0', 'nwlayers': 32, 'channelrange': '%i %i' % (startChan, endChan), 'channelsout': nsubbands, 'joinpolarizations': None } for msFile in sorted(v.MS_List): x.sh('export LC_NUMERIC=en_GB.utf8') msBase = msFile.split('/')[-1].split('.MS')[0] lsm2 = filter(lambda x: x.split('/')[-1].startswith(msBase), lsm2files)[0] v.LSM = lsm2 v.MS = msFile stefcal.STEFCAL_STEP_INCR = 0 # we set the step counter explicitly info( "########## second calibration of %s using sky model %s step %i" % (msFile, lsm2, v.STEP)) stefcal.stefcal(output='CORR_DATA', dirty=False, restore=False, section='calico-stefcal') wscleanDict['name'] = msBase + '_s%i' % v.STEP run_wsclean(msFile, wscleanDict) fitsfiles = glob.glob('*.fits') for ff in fitsfiles: shutil.move(ff, v.DESTDIR) #clean up if goto_step <= 5: gaulfiles = glob.glob('*.gaul') for gf in gaulfiles: shutil.move(gf, v.DESTDIR)
#!/usr/bin/python # uses pybdsm to generate respective catalogs for all our fits files #importing Pybdsm,Pyxis and Tigger import os import Tigger from lofar import bdsm import Pyxis import lsm #storing all the fits files in a list filenames = ['PKS1510-089_20140110_hh.fits','PKS1510-089_20140112_hh.fits','PKS1510-089_20140501_hh.fits','PKS1510-089_20140521_hh.fits','PKS1510-089_20140524_hh.fits','PKS1510-089_20140603_hh.fits','PKS1510-089_20140614_hh.fits','PKS1510-089_20140622_hh.fits','PKS1510-089_20140625_hh.fits'] i = 0 #processing the images using bdsm for f in filenames: lsm.pybdsm_search(image=f,thresh_isl=3,thresh_pix = 7,clobber = True) os.system("mv /home/ulrich/Workspace/plotsspw0/spw0-s1-_pybdsm.lsm.gaul /home/ulrich/Workspace/plotsspw0/image-%d.lsm.gaul"%(i)) os.system("mv /home/ulrich/Workspace/plotsspw0/spw0-s1-_pybdsm.lsm.html /home/ulrich/Workspace/plotsspw0/image-%d.lsm.html"%(i)) i = i + 1