def clean(p, MSs, size=2., apply_beam=False): """ p = patch name mss = list of mss to clean size = in deg of the image """ # set pixscale and imsize pixscale = MSs.getListObj()[0].getResolution() / 3. imsize = int(size / (pixscale / 3600.)) if imsize < 512: imsize = 512 if imsize % 2 == 1: imsize += 1 # make even logger.debug('Image size: ' + str(imsize) + ' - Pixel scale: ' + str(pixscale)) if apply_beam: idg_parms = '-use-idg -grid-with-beam -use-differential-lofar-beam -beam-aterm-update 400' else: idg_params = '-use-idg' # clean 1 logger.info('Cleaning (' + str(p) + ')...') imagename = 'img/ddcal-' + str(p) s.add('wsclean -reorder -temp-dir /dev/shm -name ' + imagename + ' -size '+str(imsize)+' '+str(imsize)+' -j '+str(s.max_processors)+' \ -scale ' +str(pixscale)+'arcsec -weight briggs -0.5 -niter 100000 -no-update-model-required -minuv-l 30 -mgain 0.85 -clean-border 1 \ -auto-threshold 20 ' +idg_parms+' \ -join-channels -fit-spectral-pol 2 -channels-out 10 -save-source-list ' +MSs.getStrWsclean(), \ log='wsclean-'+str(p)+'.log', commandType='wsclean', processors='max') s.run(check=True) # make mask im = lib_img.Image(imagename + '-MFS-image.fits', userReg=userReg) os.system('mv %s %s' % (im.skymodel, im.skymodel + '-first')) # copy the source list im.makeMask(threshisl=3) # clean 2 #-multiscale -multiscale-scale-bias 0.5 \ #-auto-mask 3 -rms-background-window 40 -rms-background-method rms-with-min \ logger.info('Cleaning w/ mask (' + str(p) + ')...') s.add('wsclean -continue -reorder -temp-dir /dev/shm -name ' + imagename + ' -size '+str(imsize)+' '+str(imsize)+' -j '+str(s.max_processors)+' \ -scale ' +str(pixscale)+'arcsec -weight briggs -0.5 -niter 1000000 -no-update-model-required -minuv-l 30 -mgain 0.85 -clean-border 1 \ -auto-threshold 0.1 -fits-mask ' +im.maskname+' '+idg_parms+' \ -join-channels -fit-spectral-pol 2 -channels-out 10 -save-source-list ' +MSs.getStrWsclean(), \ log='wscleanM-'+str(p)+'.log', commandType='wsclean', processors='max') s.run(check=True) os.system('cat logs/wscleanM-' + str(p) + '.log | grep "background noise"') os.system('grep -v \'^Format\' %s >> %s' % (im.skymodel + '-first', im.skymodel)) # merge the source lists lib_util.check_rm(im.skymodel + '-first') return imagename
weight='briggs -0.3', niter=1000000, no_update_model_required='', minuv_l=30, parallel_gridding=2, baseline_averaging='', maxuv_l=4500, mgain=0.85, parallel_deconvolution=512, local_rms='', auto_threshold=4, join_channels='', fit_spectral_pol=cc_fit_order, channels_out=MSs.getChout(4.e6), deconvolution_channels=cc_fit_order) im = lib_img.Image(imagename + '-MFS-image.fits', userReg=userReg) im.makeMask(threshpix=5) kwargs = { 'do_predict': True, 'reuse_dirty': imagename, 'reuse_psf': imagename } else: kwargs = {} lib_util.run_wsclean(s, 'wscleanM-c' + str(c) + '.log', MSs.getStrWsclean(), name=imagenameM, save_source_list='',
# set image size at 1.5 * FWHM imgsizepix = int(1.5 * MSs.getListObj()[0].getFWHM() / (2. / 3600)) imgsizepix += imgsizepix % 2 # make even if c >= 2: imgsizepix *= 2 # last cycle make a very large image to catch source in the sidelobes # clean mask clean logger.info('Cleaning (cycle: ' + str(c) + ')...') imagename = 'img/wide-' + str(c) lib_util.run_wsclean(s, 'wscleanA-c'+str(c)+'.log', MSs.getStrWsclean(), name=imagename, size=imgsizepix, scale='2arcsec', \ weight='briggs 0.', niter=10000, no_update_model_required='', mgain=0.9, \ baseline_averaging=5, parallel_deconvolution=256, auto_threshold=20, \ join_channels='', fit_spectral_pol=2, channels_out=8) # make mask im = lib_img.Image(imagename + '-MFS-image.fits', userReg=userReg) im.makeMask(threshisl=4, atrous_do=False) # baseline averaging possible as we cut longest baselines (also it is in time, where smearing is less problematic) # TODO: add -parallel-deconvolution=256 when source lists can be saved (https://sourceforge.net/p/wsclean/tickets/141/) logger.info('Cleaning w/ mask (cycle: ' + str(c) + ')...') imagename = 'img/wideM-' + str(c) lib_util.run_wsclean(s, 'wscleanB-c'+str(c)+'.log', MSs.getStrWsclean(), name=imagename, save_source_list='', size=imgsizepix, scale='2arcsec', \ weight='briggs 0.', niter=100000, no_update_model_required='', mgain=0.9, \ #multiscale='', multiscale_scales='0,5,10,20,40', \ baseline_averaging=5, auto_threshold=1, fits_mask=im.maskname, \ join_channels='', fit_spectral_pol=2, channels_out=8) os.system('cat logs/wscleanB-c' + str(c) + '.log | grep "background noise"') if c != 2:
def clean(p, MSs, size, res='normal', apply_beam=False): """ p = patch name mss = list of mss to clean size = in deg of the image """ # set pixscale and imsize pixscale = MSs.resolution if res == 'normal': pixscale = float('%.1f' % (pixscale / 2.5)) elif res == 'high': pixscale = float('%.1f' % (pixscale / 3.5)) elif res == 'low': pass # no change imsize = [0, 0] imsize[0] = int(size[0] * 1.1 / (pixscale / 3600.)) # add 10% imsize[1] = int(size[1] * 1.1 / (pixscale / 3600.)) # add 10% imsize[0] += imsize[0] % 2 imsize[1] += imsize[1] % 2 if imsize[0] < 64: imsize[0] == 64 if imsize[1] < 64: imsize[1] == 64 logger.debug('Image size: ' + str(imsize) + ' - Pixel scale: ' + str(pixscale)) if res == 'normal': weight = 'briggs -0.1' maxuv_l = None elif res == 'high': weight = 'briggs -0.6' maxuv_l = None elif res == 'low': weight = 'briggs 0' maxuv_l = 3500 # clean 1 logger.info('Cleaning (' + str(p) + ')...') imagename = 'img/ddcal-' + str(p) lib_util.run_wsclean(s, 'wscleanA-'+str(p)+'.log', MSs.getStrWsclean(), name=imagename, size=imsize, scale=str(pixscale)+'arcsec', \ weight=weight, niter=10000, no_update_model_required='', minuv_l=30, maxuv_l=maxuv_l, mgain=0.85, \ baseline_averaging='', parallel_deconvolution=512, auto_threshold=5, \ join_channels='', fit_spectral_pol=3, channels_out=9, deconvolution_channels=3) # make mask im = lib_img.Image(imagename + '-MFS-image.fits', userReg=userReg) im.makeMask(threshpix=3) # clean 2 logger.info('Cleaning w/ mask (' + str(p) + ')...') imagename = 'img/ddcalM-' + str(p) if apply_beam: lib_util.run_wsclean(s, 'wscleanB-'+str(p)+'.log', MSs.getStrWsclean(), name=imagename, save_source_list='', size=imsize, scale=str(pixscale)+'arcsec', \ weight=weight, niter=100000, no_update_model_required='', minuv_l=30, maxuv_l=maxuv_l, mgain=0.85, \ use_idg='', grid_with_beam='', use_differential_lofar_beam='', beam_aterm_update=400, \ multiscale='', multiscale_scales='0,10,20,40,80', \ parallel_deconvolution=512, local_rms='', auto_threshold=0.75, auto_mask=1.5, fits_mask=im.maskname, \ join_channels='', fit_spectral_pol=3, channels_out=9, deconvolution_channels=3) else: lib_util.run_wsclean(s, 'wscleanB-'+str(p)+'.log', MSs.getStrWsclean(), name=imagename, size=imsize, save_source_list='', scale=str(pixscale)+'arcsec', \ weight=weight, niter=100000, no_update_model_required='', minuv_l=30, maxuv_l=maxuv_l, mgain=0.85, \ multiscale='', multiscale_scales='0,10,20,40,80', baseline_averaging='', parallel_deconvolution=512, local_rms='', auto_threshold=0.75, auto_mask=1.5, fits_mask=im.maskname, \ join_channels='', fit_spectral_pol=3, channels_out=9, deconvolution_channels=3) os.system('cat logs/wscleanA-' + str(p) + '.log logs/wscleanB-' + str(p) + '.log | grep "background noise"')
os.makedirs('mss-dd') MSs_self.run('DPPP '+parset_dir+'/DPPP-avg.parset msin=$pathMS msout=mss-dd/$nameMS.MS msin.datacolumn=CORRECTED_DATA avg.freqstep=1 avg.timestep=1', \ log='$nameMS_avg.log', commandType='DPPP') MSs = lib_ms.AllMSs(glob.glob('mss-dd/TC*[0-9].MS'), s) logger.info('Add columns...') MSs.run('addcol2ms.py -m $pathMS -c CORRECTED_DATA,SUBTRACTED_DATA -i DATA', log='$nameMS_addcol.log', commandType='python') ############################################################## # setup initial model MSs.getListObj()[0].makeBeamReg('ddcal/beam.reg', freq='mid') beamReg = 'ddcal/beam.reg' mosaic_image = lib_img.Image(sorted( glob.glob('self/images/wideM-[0-9]-MFS-image.fits'))[-1], userReg=userReg) mosaic_image.selectCC() # TEST: #mosaic_image = lib_img.Image('ddcal/images/c00/mos-MFS-image.fits', userReg = userReg) rms_noise_pre = np.inf for c in range(maxniter): logger.info('Starting cycle: %i' % c) if c >= 1: directions_old = directions with w.if_todo('delimg-c%02i' % c): lib_util.check_rm('img') os.makedirs('img')
taper_gaussian='30arcsec', niter=1000000, no_update_model_required='', minuv_l=30, mgain=0.75, nmiter=0, auto_threshold=5, local_rms='', local_rms_method='rms-with-min', join_channels='', fit_spectral_pol=2, channels_out=2) os.system('cat logs/wsclean-wide.log | grep "background noise"') # makemask im = lib_img.Image(imagename + '-MFS-image.fits', userReg=region) im.makeMask(threshpix=5, rmsbox=(50, 5)) maskfits = imagename + '-mask.fits' logger.info('Cleaning wide 2...') imagenameM = 'img/img-wideM' lib_util.run_wsclean(s, 'wsclean-wide.log', MSs.getStrWsclean(), name=imagenameM, do_predict=True, parallel_gridding=4, baseline_averaging='', size=2500, reuse_psf=imagename, reuse_dirty=imagename,
imagename = 'img/calLR' lib_util.run_wsclean(s, 'wscleanLR.log', MSs.getStrWsclean(), name=imagename, size=imgsizepix/5, scale='60arcsec', \ weight='briggs 0.', taper_gaussian='240arcsec', niter=10000, no_update_model_required='', minuv_l=30, maxuv_l=2000, mgain=0.85, \ use_idg='', grid_with_beam='', use_differential_lofar_beam='', beam_aterm_update=400, \ parallel_deconvolution=512, \ auto_mask=10, auto_threshold=1, pol='IQUV', join_polarizations='', join_channels='', fit_spectral_pol=3, channels_out=9, deconvolution_channels=3) logger.info('Cleaning normal...') imagename = 'img/cal' lib_util.run_wsclean(s, 'wscleanA.log', MSs.getStrWsclean(), name=imagename, size=imgsizepix, scale='5arcsec', \ weight='briggs 0.', niter=10000, no_update_model_required='', minuv_l=30, mgain=0.85, \ baseline_averaging='', parallel_deconvolution=512, \ auto_threshold=20, join_channels='', fit_spectral_pol=3, channels_out=9, deconvolution_channels=3) # make mask im = lib_img.Image(imagename + '-MFS-image.fits') im.makeMask(threshisl=3) logger.info('Cleaning w/ mask...') lib_util.run_wsclean(s, 'wscleanB.log', MSs.getStrWsclean(), name=imagename, size=imgsizepix, scale='5arcsec', \ weight='briggs 0.', niter=100000, no_update_model_required='', minuv_l=30, mgain=0.85, \ baseline_averaging='', parallel_deconvolution=512, \ auto_threshold=0.1, fits_mask=im.maskname, join_channels='', fit_spectral_pol=3, channels_out=9, deconvolution_channels=3) os.system( 'cat logs/wscleanA.log logs/wscleanB.log | grep "background noise"') # make new mask im.makeMask(threshisl=5) logger.info("Done.")
def clean(p, MSs, res='normal', size=[1, 1], empty=False, imagereg=None): """ p = patch name mss = list of mss to clean size = in deg of the image """ # set pixscale and imsize pixscale = MSs.resolution if res == 'normal': pixscale = float('%.1f' % (pixscale / 2.5)) elif res == 'high': pixscale = float('%.1f' % (pixscale / 3.5)) elif res == 'low': pass # no change imsize = [ int(size[0] * 1.5 / (pixscale / 3600.)), int(size[1] * 1.5 / (pixscale / 3600.)) ] # add 50% imsize[0] += imsize[0] % 2 imsize[1] += imsize[1] % 2 if imsize[0] < 256: imsize[0] = 256 if imsize[1] < 256: imsize[1] = 256 logger.debug('Image size: ' + str(imsize) + ' - Pixel scale: ' + str(pixscale)) if res == 'normal': weight = 'briggs -0.3' maxuv_l = None elif res == 'high': weight = 'briggs -0.6' maxuv_l = None elif res == 'low': weight = 'briggs 0' maxuv_l = 3500 else: logger.error('Wrong "res": %s.' % str(res)) sys.exit() if empty: logger.info('Cleaning empty (' + str(p) + ')...') imagename = 'img/empty-' + str(p) lib_util.run_wsclean(s, 'wscleanE-' + str(p) + '.log', MSs.getStrWsclean(), name=imagename, data_column='SUBTRACTED_DATA', size=imsize, scale=str(pixscale) + 'arcsec', weight=weight, niter=0, no_update_model_required='', minuv_l=30, mgain=0, baseline_averaging='') else: # clean 1 logger.info('Cleaning (' + str(p) + ')...') imagename = 'img/extract-' + str(p) lib_util.run_wsclean(s, 'wscleanA-' + str(p) + '.log', MSs.getStrWsclean(), name=imagename, size=imsize, scale=str(pixscale) + 'arcsec', weight=weight, niter=10000, no_update_model_required='', minuv_l=30, maxuv_l=maxuv_l, mgain=0.85, baseline_averaging='', parallel_deconvolution=512, auto_threshold=5, join_channels='', fit_spectral_pol=3, channels_out=ch_out, deconvolution_channels=3) # make mask im = lib_img.Image(imagename + '-MFS-image.fits', userReg=userReg) try: im.makeMask(threshpix=10, rmsbox=(70, 5)) except: logger.warning('Fail to create mask for %s.' % imagename + '-MFS-image.fits') return if imagereg is not None: lib_img.blank_image_reg(im.maskname, imagereg, inverse=True, blankval=0.) # clean 2 # TODO: add deconvolution_channels when bug fixed logger.info('Cleaning w/ mask (' + str(p) + ')...') imagenameM = 'img/extractM-' + str(p) lib_util.run_wsclean( s, 'wscleanB-' + str(p) + '.log', MSs.getStrWsclean(), name=imagenameM, do_predict=True, size=imsize, scale=str(pixscale) + 'arcsec', weight=weight, niter=100000, no_update_model_required='', minuv_l=30, maxuv_l=maxuv_l, reuse_psf=imagename, reuse_dirty=imagename, mgain=0.85, multiscale='', baseline_averaging='', parallel_deconvolution=512, auto_threshold=0.7, auto_mask=1.5, fits_mask=im.maskname, join_channels='', fit_spectral_pol=3, channels_out=ch_out) # , deconvolution_channels=3) os.system('cat logs/wscleanB-' + str(p) + '.log | grep "background noise"')
os.system( 'cp ddcal/c01/solutions/interp.h5 extract/init/') # copy fnal dde sol lib_util.check_rm('mss-extract') if not os.path.exists('mss-extract'): logger.info('Copy MS...') os.system('cp -r mss-avg mss-extract') # region must be a list of ds9 circles and polygons (other shapes can be implemented in lib_util.Rgion_helper() target_reg = lib_util.Region_helper(target_reg_file) center = target_reg.get_center() # center of the extract region MSs = lib_ms.AllMSs(glob.glob('mss-extract/*MS'), s) ch_out = MSs.getChout(4e6) # chout from dd-serial fwhm = MSs.getListObj()[0].getFWHM(freq='mid') phase_center = MSs.getListObj()[0].getPhaseCentre() # read image, h5parm, make mask wideDD_image = lib_img.Image('extract/init/wideDD-c01.app.restored.fits') dde_h5parm = 'extract/init/interp.h5' # make mask for subtraction mask_ddcal = wideDD_image.imagename.replace( '.fits', '_mask-ddcal.fits') # this is used to find calibrators wideDD_image.makeMask(threshpix=5, atrous_do=True, maskname=mask_ddcal, write_srl=True, write_ds9=True) def clean(p, MSs, res='normal', size=[1, 1], empty=False, imagereg=None): """ p = patch name mss = list of mss to clean
lofar_system = 'hba' flag_steps = "[ears, ant, uvmin, elev, count]" ######################################################## # flag bad stations, and low-elev logger.info('Flagging...') MSs.run('DPPP '+parset_dir+'/DPPP-flag.parset msin=$pathMS msout=. steps=\"'+flag_steps+'\" ant.baseline=\"'+bl2flag+'\"', \ log='$nameMS_flag.log', commandType='DPPP') if lofar_system == 'hba': model_dir = '/home/fdg/scripts/model/AteamHBA/' + patch else: model_dir = '/home/fdg/scripts/model/AteamLBA/' + patch if os.path.exists(model_dir + '/img-MFS-model.fits'): im = lib_img.Image(model_dir + '/img-MFS-image.fits') im.rescaleModel(f) n = len(glob.glob(model_dir + '/img-[0-9]*-model.fits')) logger.info('Predict (wsclean: %s - chan: %i)...' % (model_dir, n)) s.add('wsclean -predict -name '+model_dir+'/img -j '+str(s.max_processors)+' -channels-out '+str(n)+' '+MSs.getStrWsclean(), \ log='wscleanPRE-init.log', commandType='wsclean', processors='max') s.run(check=True) else: logger.info('Predict (DPPP)...') MSs.run('DPPP ' + parset_dir + '/DPPP-predict.parset msin=$pathMS pre.sourcedb=' + skymodel + ' pre.sources=' + patch, log='$nameMS_pre.log', commandType='DPPP') # TESTTESTTEST
if imgsizepix % 2 != 0: imgsizepix += 1 # prevent odd img sizes #MSs.getListObj()[0].makeBeamReg('ddcal/beam.reg', freq='mid') #beamReg = 'ddcal/beam.reg' logger.info('Add columns...') MSs.run('addcol2ms.py -m $pathMS -c CORRECTED_DATA,SUBTRACTED_DATA -i DATA', log='$nameMS_addcol.log', commandType='python') MSs.run('addcol2ms.py -m $pathMS -c FLAG_BKP -i FLAG', log='$nameMS_addcol.log', commandType='python') ############################################################## # setup initial model os.system('cp self/images/wideM-1* ddcal/init/') full_image = lib_img.Image('ddcal/init/wideM-1-MFS-image.fits', userReg=userReg) for cmaj in range(maxIter): logger.info('Starting major cycle: %i' % cmaj) # cycle specific variables picklefile = 'ddcal/directions-c%02i.pickle' % cmaj interp_h5parm = 'ddcal/c%02i/solutions/interp.h5' % cmaj #aterm_config_file = 'ddcal/c%02i/aterm/aterm.config' % cmaj mask_ddcal = full_image.imagename.replace( '.fits', '_mask-ddcal.fits') # this is used to find calibrators if not os.path.exists('ddcal/c%02i' % cmaj): os.makedirs('ddcal/c%02i' % cmaj) for subdir in ['plots', 'images', 'solutions', 'skymodels']: if not os.path.exists('ddcal/c%02i/%s' % (cmaj, subdir)):
target_reg = 'target.reg' ####################################################### from LiLF import lib_ms, lib_img, lib_util, lib_log, lib_dd logger_obj = lib_log.Logger('pipeline-facet_self.logger') logger = lib_log.logger s = lib_util.Scheduler(log_dir=logger_obj.log_dir, dry=False) #, maxThreads = 4) w = lib_util.Walker('pipeline-facet-self.walker') # parse parset parset = lib_util.getParset() parset_dir = parset.get('LOFAR_facet_self', 'parset_dir') maxniter = parset.getint('LOFAR_facet_self', 'maxniter') userReg = parset.get('model', 'userReg') mosaic_image = lib_img.Image('ddcal/images/c%02i/mos-MFS-image.fits' % lastcycle) ############################ if w.todo('cleaning'): logger.info('Cleaning...') lib_util.check_rm('plot*') lib_util.check_rm('cal*h5') lib_util.check_rm('img') lib_util.check_rm('facet') os.makedirs('img') os.makedirs('facet') lib_util.check_rm('mss-facet') if not os.path.exists('mss-facet'): os.system('cp -r mss-dd mss-facet') w.done('cleaning')
Ss = lib_ms.AllMSs(glob.glob('mss-dd/TC*[0-9].MS'), s) logger.info('Add columns...') MSs.run('addcol2ms.py -m $pathMS -c CORRECTED_DATA,SUBTRACTED_DATA', log='$nameMS_addcol.log', commandType='python') ############################################################## logger.info('BL-based smoothing...') MSs.run('BLsmooth.py -f 1.0 -r -i DATA -o SMOOTHED_DATA $pathMS', log='$nameMS_smooth.log', commandType='python') # setup initial model mosaic_image = lib_img.Image(sorted( glob.glob('self/images/wide-[0-9]-MFS-image.fits'))[-1], userReg=userReg) mosaic_image.selectCC() rms_noise_pre = np.inf for c in xrange(maxniter): logger.info('Starting cycle: %i' % c) lib_util.check_rm('img') os.makedirs('img') os.makedirs('ddcal/images/c%02i/regions' % c) mask = 'ddcal/masks/facets%02i.fits' % c ### group into patches of similar flux lsm = lsmtool.load(mosaic_image.skymodel_cut) lsm.group('tessellate',
s.run(check=True) # clean mask clean (cut at 5k lambda) logger.info('Cleaning (cycle: ' + str(c) + ')...') imagename = 'img/wide-' + str(c) s.add('wsclean -reorder -temp-dir '+ temp_dir +' -name ' + imagename + ' -size ' + str(size) + ' ' + str(size) + ' -j '+str(s.max_processors)+' \ -scale 10arcsec -weight briggs 0. -niter 100000 -update-model-required -minuv-l 30 -maxuv-l 5000 -mgain 0.85 -clean-border 1 \ -multiscale -multiscale-scale-bias 0.5 -multiscale-scales 0,4,16 \ -auto-threshold 20 \ -use-idg \ -join-channels -fit-spectral-pol 2 -channels-out 10 ' +MSs.getStrWsclean(), \ log='wsclean-c'+str(c)+'.log', commandType='wsclean', processors='max') s.run(check=True) # make mask im = lib_img.Image(imagename + '-MFS-image.fits', userReg=userReg) im.makeMask(threshisl=3) logger.info('Cleaning w/ mask (cycle: ' + str(c) + ')...') s.add('wsclean -continue -reorder -temp-dir '+ temp_dir +' -name ' + imagename + ' -size ' + str(size) + ' ' + str(size) + ' -j '+str(s.max_processors)+' \ -scale 10arcsec -weight briggs 0. -niter 200000 -update-model-required -minuv-l 30 -maxuv-l 5000 -mgain 0.85 -clean-border 1 \ -multiscale -multiscale-scale-bias 0.5 -multiscale-scales 0,4,16 \ -auto-threshold 1 -fits-mask ' +im.maskname+' \ -use-idg \ -join-channels -fit-spectral-pol 2 -channels-out 10 -save-source-list ' +MSs.getStrWsclean(), \ log='wscleanM-c'+str(c)+'.log', commandType='wsclean', processors='max') s.run(check=True) os.system('cat logs/wscleanM-c' + str(c) + '.log | grep "background noise"') im = lib_img.Image(imagename + '-MFS-image.fits',
def clean(p, MSs, size, res='normal', apply_beam=False): """ p = patch name mss = list of mss to clean size = in deg of the image """ # set pixscale and imsize pixscale = MSs.getListObj()[0].getResolution() if res == 'normal': pixscale = float('%.1f' % (pixscale / 2.)) elif res == 'high': pixscale = float('%.1f' % (pixscale / 4.)) elif res == 'low': pass # no change imsize = [0, 0] imsize[0] = int(size[0] * 1.05 / (pixscale / 3600.)) # add 5% imsize[1] = int(size[1] * 1.05 / (pixscale / 3600.)) # add 5% imsize[0] += imsize[0] % 2 imsize[1] += imsize[1] % 2 if imsize[0] < 64: imsize[0] == 64 if imsize[1] < 64: imsize[1] == 64 logger.debug('Image size: ' + str(imsize) + ' - Pixel scale: ' + str(pixscale)) if res == 'normal': weight = 'briggs 0' maxuv_l = None elif res == 'high': weight = 'briggs -0.7' maxuv_l = None elif res == 'low': weight = 'briggs 0' maxuv_l = 3500 # clean 1 logger.info('Cleaning (' + str(p) + ')...') imagename = 'img/ddcal-' + str(p) lib_util.run_wsclean(s, 'wscleanA-'+str(p)+'.log', MSs.getStrWsclean(), name=imagename, size=imsize, scale=str(pixscale)+'arcsec', \ weight=weight, niter=10000, no_update_model_required='', minuv_l=30, maxuv_l=maxuv_l, mgain=0.8, \ baseline_averaging=5, parallel_deconvolution=256, auto_threshold=3, \ join_channels='', fit_spectral_pol=3, channels_out=9, deconvolution_channels=3) # make mask im = lib_img.Image(imagename + '-MFS-image.fits', userReg=userReg) im.makeMask(threshisl=4) # clean 2 # TODO: add -parallel-deconvolution when source lists can be saved (https://sourceforge.net/p/wsclean/tickets/141/) logger.info('Cleaning w/ mask (' + str(p) + ')...') imagename = 'img/ddcalM-' + str(p) if apply_beam: lib_util.run_wsclean(s, 'wscleanB-'+str(p)+'.log', MSs.getStrWsclean(), name=imagename, save_source_list='', size=imsize, scale=str(pixscale)+'arcsec', \ weight=weight, niter=100000, no_update_model_required='', minuv_l=30, maxuv_l=maxuv_l, mgain=0.8, \ use_idg='', grid_with_beam='', use_differential_lofar_beam='', beam_aterm_update=400, \ multiscale='', \ auto_threshold=0.5, fits_mask=im.maskname, \ join_channels='', fit_spectral_pol=3, channels_out=9, deconvolution_channels=3) logger.info('Cleaning V (' + str(p) + ')...') imagename = 'img/ddcalV-' + str(p) lib_util.run_wsclean(s, 'wscleanV-'+str(p)+'.log', MSs.getStrWsclean(), name=imagename, size=imgsize, scale=str(pixscale)+'srcsec', pol='V', \ weight='briggs 0.', niter=1000, no_update_model_required='', minuv_l=30, maxuv_l=5000, \ baseline_averaging=5) else: lib_util.run_wsclean(s, 'wscleanB-'+str(p)+'.log', MSs.getStrWsclean(), name=imagename, size=imsize, save_source_list='', scale=str(pixscale)+'arcsec', \ weight=weight, niter=50000, no_update_model_required='', minuv_l=30, maxuv_l=maxuv_l, mgain=0.85, \ multiscale='', \ auto_threshold=0.5, fits_mask=im.maskname, \ baseline_averaging=5, join_channels='', fit_spectral_pol=3, channels_out=9, deconvolution_channels=3) os.system('cat logs/wscleanA-' + str(p) + '.log logs/wscleanB-' + str(p) + '.log | grep "background noise"')
logger.info('Sub model...') MSs.run( 'taql "update $pathMS set CORRECTED_DATA = CORRECTED_DATA - MODEL_DATA"', log='$nameMS_taql1.log', commandType='general') #logger.info('Copy MODEL_DATA...') #MSs.run('taql "update $pathMS set MODEL_DATA_HIGHRES = MODEL_DATA"', log='$nameMS_taql2.log', commandType='general') logger.info('Cleaning sub (cycle %i)...' % c) imagename = 'img/imgsub-c' + str(c) s.add('wsclean -reorder -temp-dir '+ temp_dir +' -name ' + imagename + ' -size 1000 1000 -j '+str(s.max_processors)+' \ -scale 15arcsec -weight briggs -1.0 -taper-gaussian 80arcsec -niter 10000 -no-update-model-required -minuv-l 30 -mgain 0.85 -clean-border 1 \ -multiscale -multiscale-scales 0,4,8,16 \ -use-idg \ -join-channels -fit-spectral-pol 2 -channels-out 15 ' +MSs.getStrWsclean(), \ log='wscleanSUB-c'+str(c)+'.log', commandType='wsclean', processors='max') s.run(check=True) #logger.info('Combining MODEL_DATA_HIGHRES and MODEL_DATA...') #MSs.run('taql "update $pathMS set MODEL_DATA = MODEL_DATA_HIGHRES + MODEL_DATA"', log='$nameMS_taql3.log', commandType='general') imagename = 'img/img-c' + str(c) im = lib_img.Image(imagename) im.rescaleModel(f) logger.info('Predict (wsclean)...') s.add('wsclean -predict -name '+imagename+' -j '+str(s.max_processors)+' -channelsout 15 '+MSs.getStrWsclean(), \ log='wscleanPRE-c'+str(c)+'.log', commandType='wsclean', processors='max') s.run(check=True) logger.info("Done.")