def clean(p, MSs, size=2., apply_beam=False):
"""
p = patch name
mss = list of mss to clean
size = in deg of the image
"""
# set pixscale and imsize
pixscale = MSs.getListObj()[0].getResolution() / 3.
imsize = int(size / (pixscale / 3600.))
if imsize < 512:
imsize = 512
if imsize % 2 == 1: imsize += 1 # make even
logger.debug('Image size: ' + str(imsize) + ' - Pixel scale: ' +
str(pixscale))
if apply_beam:
idg_parms = '-use-idg -grid-with-beam -use-differential-lofar-beam -beam-aterm-update 400'
else:
idg_params = '-use-idg'
# clean 1
logger.info('Cleaning (' + str(p) + ')...')
imagename = 'img/ddcal-' + str(p)
s.add('wsclean -reorder -temp-dir /dev/shm -name ' + imagename + ' -size '+str(imsize)+' '+str(imsize)+' -j '+str(s.max_processors)+' \
-scale ' +str(pixscale)+'arcsec -weight briggs -0.5 -niter 100000 -no-update-model-required -minuv-l 30 -mgain 0.85 -clean-border 1 \
-auto-threshold 20 ' +idg_parms+' \
-join-channels -fit-spectral-pol 2 -channels-out 10 -save-source-list ' +MSs.getStrWsclean(), \
log='wsclean-'+str(p)+'.log', commandType='wsclean', processors='max')
s.run(check=True)
# make mask
im = lib_img.Image(imagename + '-MFS-image.fits', userReg=userReg)
os.system('mv %s %s' %
(im.skymodel, im.skymodel + '-first')) # copy the source list
im.makeMask(threshisl=3)
# clean 2
#-multiscale -multiscale-scale-bias 0.5 \
#-auto-mask 3 -rms-background-window 40 -rms-background-method rms-with-min \
logger.info('Cleaning w/ mask (' + str(p) + ')...')
s.add('wsclean -continue -reorder -temp-dir /dev/shm -name ' + imagename + ' -size '+str(imsize)+' '+str(imsize)+' -j '+str(s.max_processors)+' \
-scale ' +str(pixscale)+'arcsec -weight briggs -0.5 -niter 1000000 -no-update-model-required -minuv-l 30 -mgain 0.85 -clean-border 1 \
-auto-threshold 0.1 -fits-mask ' +im.maskname+' '+idg_parms+' \
-join-channels -fit-spectral-pol 2 -channels-out 10 -save-source-list ' +MSs.getStrWsclean(), \
log='wscleanM-'+str(p)+'.log', commandType='wsclean', processors='max')
s.run(check=True)
os.system('cat logs/wscleanM-' + str(p) + '.log | grep "background noise"')
os.system('grep -v \'^Format\' %s >> %s' %
(im.skymodel + '-first', im.skymodel)) # merge the source lists
lib_util.check_rm(im.skymodel + '-first')
return imagename
weight='briggs -0.3',
niter=1000000,
no_update_model_required='',
minuv_l=30,
parallel_gridding=2,
baseline_averaging='',
maxuv_l=4500,
mgain=0.85,
parallel_deconvolution=512,
local_rms='',
auto_threshold=4,
join_channels='',
fit_spectral_pol=cc_fit_order,
channels_out=MSs.getChout(4.e6),
deconvolution_channels=cc_fit_order)
im = lib_img.Image(imagename + '-MFS-image.fits', userReg=userReg)
im.makeMask(threshpix=5)
kwargs = {
'do_predict': True,
'reuse_dirty': imagename,
'reuse_psf': imagename
}
else:
kwargs = {}
lib_util.run_wsclean(s,
'wscleanM-c' + str(c) + '.log',
MSs.getStrWsclean(),
name=imagenameM,
save_source_list='',
# set image size at 1.5 * FWHM
imgsizepix = int(1.5 * MSs.getListObj()[0].getFWHM() / (2. / 3600))
imgsizepix += imgsizepix % 2 # make even
if c >= 2:
imgsizepix *= 2 # last cycle make a very large image to catch source in the sidelobes
# clean mask clean
logger.info('Cleaning (cycle: ' + str(c) + ')...')
imagename = 'img/wide-' + str(c)
lib_util.run_wsclean(s, 'wscleanA-c'+str(c)+'.log', MSs.getStrWsclean(), name=imagename, size=imgsizepix, scale='2arcsec', \
weight='briggs 0.', niter=10000, no_update_model_required='', mgain=0.9, \
baseline_averaging=5, parallel_deconvolution=256, auto_threshold=20, \
join_channels='', fit_spectral_pol=2, channels_out=8)
# make mask
im = lib_img.Image(imagename + '-MFS-image.fits', userReg=userReg)
im.makeMask(threshisl=4, atrous_do=False)
# baseline averaging possible as we cut longest baselines (also it is in time, where smearing is less problematic)
# TODO: add -parallel-deconvolution=256 when source lists can be saved (https://sourceforge.net/p/wsclean/tickets/141/)
logger.info('Cleaning w/ mask (cycle: ' + str(c) + ')...')
imagename = 'img/wideM-' + str(c)
lib_util.run_wsclean(s, 'wscleanB-c'+str(c)+'.log', MSs.getStrWsclean(), name=imagename, save_source_list='', size=imgsizepix, scale='2arcsec', \
weight='briggs 0.', niter=100000, no_update_model_required='', mgain=0.9, \
#multiscale='', multiscale_scales='0,5,10,20,40', \
baseline_averaging=5, auto_threshold=1, fits_mask=im.maskname, \
join_channels='', fit_spectral_pol=2, channels_out=8)
os.system('cat logs/wscleanB-c' + str(c) +
'.log | grep "background noise"')
if c != 2:
def clean(p, MSs, size, res='normal', apply_beam=False):
"""
p = patch name
mss = list of mss to clean
size = in deg of the image
"""
# set pixscale and imsize
pixscale = MSs.resolution
if res == 'normal':
pixscale = float('%.1f' % (pixscale / 2.5))
elif res == 'high':
pixscale = float('%.1f' % (pixscale / 3.5))
elif res == 'low':
pass # no change
imsize = [0, 0]
imsize[0] = int(size[0] * 1.1 / (pixscale / 3600.)) # add 10%
imsize[1] = int(size[1] * 1.1 / (pixscale / 3600.)) # add 10%
imsize[0] += imsize[0] % 2
imsize[1] += imsize[1] % 2
if imsize[0] < 64: imsize[0] == 64
if imsize[1] < 64: imsize[1] == 64
logger.debug('Image size: ' + str(imsize) + ' - Pixel scale: ' +
str(pixscale))
if res == 'normal':
weight = 'briggs -0.1'
maxuv_l = None
elif res == 'high':
weight = 'briggs -0.6'
maxuv_l = None
elif res == 'low':
weight = 'briggs 0'
maxuv_l = 3500
# clean 1
logger.info('Cleaning (' + str(p) + ')...')
imagename = 'img/ddcal-' + str(p)
lib_util.run_wsclean(s, 'wscleanA-'+str(p)+'.log', MSs.getStrWsclean(), name=imagename, size=imsize, scale=str(pixscale)+'arcsec', \
weight=weight, niter=10000, no_update_model_required='', minuv_l=30, maxuv_l=maxuv_l, mgain=0.85, \
baseline_averaging='', parallel_deconvolution=512, auto_threshold=5, \
join_channels='', fit_spectral_pol=3, channels_out=9, deconvolution_channels=3)
# make mask
im = lib_img.Image(imagename + '-MFS-image.fits', userReg=userReg)
im.makeMask(threshpix=3)
# clean 2
logger.info('Cleaning w/ mask (' + str(p) + ')...')
imagename = 'img/ddcalM-' + str(p)
if apply_beam:
lib_util.run_wsclean(s, 'wscleanB-'+str(p)+'.log', MSs.getStrWsclean(), name=imagename, save_source_list='', size=imsize, scale=str(pixscale)+'arcsec', \
weight=weight, niter=100000, no_update_model_required='', minuv_l=30, maxuv_l=maxuv_l, mgain=0.85, \
use_idg='', grid_with_beam='', use_differential_lofar_beam='', beam_aterm_update=400, \
multiscale='', multiscale_scales='0,10,20,40,80', \
parallel_deconvolution=512, local_rms='', auto_threshold=0.75, auto_mask=1.5, fits_mask=im.maskname, \
join_channels='', fit_spectral_pol=3, channels_out=9, deconvolution_channels=3)
else:
lib_util.run_wsclean(s, 'wscleanB-'+str(p)+'.log', MSs.getStrWsclean(), name=imagename, size=imsize, save_source_list='', scale=str(pixscale)+'arcsec', \
weight=weight, niter=100000, no_update_model_required='', minuv_l=30, maxuv_l=maxuv_l, mgain=0.85, \
multiscale='', multiscale_scales='0,10,20,40,80',
baseline_averaging='', parallel_deconvolution=512, local_rms='', auto_threshold=0.75, auto_mask=1.5, fits_mask=im.maskname, \
join_channels='', fit_spectral_pol=3, channels_out=9, deconvolution_channels=3)
os.system('cat logs/wscleanA-' + str(p) + '.log logs/wscleanB-' + str(p) +
'.log | grep "background noise"')
os.makedirs('mss-dd')
MSs_self.run('DPPP '+parset_dir+'/DPPP-avg.parset msin=$pathMS msout=mss-dd/$nameMS.MS msin.datacolumn=CORRECTED_DATA avg.freqstep=1 avg.timestep=1', \
log='$nameMS_avg.log', commandType='DPPP')
MSs = lib_ms.AllMSs(glob.glob('mss-dd/TC*[0-9].MS'), s)
logger.info('Add columns...')
MSs.run('addcol2ms.py -m $pathMS -c CORRECTED_DATA,SUBTRACTED_DATA -i DATA',
log='$nameMS_addcol.log',
commandType='python')
##############################################################
# setup initial model
MSs.getListObj()[0].makeBeamReg('ddcal/beam.reg', freq='mid')
beamReg = 'ddcal/beam.reg'
mosaic_image = lib_img.Image(sorted(
glob.glob('self/images/wideM-[0-9]-MFS-image.fits'))[-1],
userReg=userReg)
mosaic_image.selectCC()
# TEST:
#mosaic_image = lib_img.Image('ddcal/images/c00/mos-MFS-image.fits', userReg = userReg)
rms_noise_pre = np.inf
for c in range(maxniter):
logger.info('Starting cycle: %i' % c)
if c >= 1: directions_old = directions
with w.if_todo('delimg-c%02i' % c):
lib_util.check_rm('img')
os.makedirs('img')
taper_gaussian='30arcsec',
niter=1000000,
no_update_model_required='',
minuv_l=30,
mgain=0.75,
nmiter=0,
auto_threshold=5,
local_rms='',
local_rms_method='rms-with-min',
join_channels='',
fit_spectral_pol=2,
channels_out=2)
os.system('cat logs/wsclean-wide.log | grep "background noise"')
# makemask
im = lib_img.Image(imagename + '-MFS-image.fits', userReg=region)
im.makeMask(threshpix=5, rmsbox=(50, 5))
maskfits = imagename + '-mask.fits'
logger.info('Cleaning wide 2...')
imagenameM = 'img/img-wideM'
lib_util.run_wsclean(s,
'wsclean-wide.log',
MSs.getStrWsclean(),
name=imagenameM,
do_predict=True,
parallel_gridding=4,
baseline_averaging='',
size=2500,
reuse_psf=imagename,
reuse_dirty=imagename,
imagename = 'img/calLR'
lib_util.run_wsclean(s, 'wscleanLR.log', MSs.getStrWsclean(), name=imagename, size=imgsizepix/5, scale='60arcsec', \
weight='briggs 0.', taper_gaussian='240arcsec', niter=10000, no_update_model_required='', minuv_l=30, maxuv_l=2000, mgain=0.85, \
use_idg='', grid_with_beam='', use_differential_lofar_beam='', beam_aterm_update=400, \
parallel_deconvolution=512, \
auto_mask=10, auto_threshold=1, pol='IQUV', join_polarizations='', join_channels='', fit_spectral_pol=3, channels_out=9, deconvolution_channels=3)
logger.info('Cleaning normal...')
imagename = 'img/cal'
lib_util.run_wsclean(s, 'wscleanA.log', MSs.getStrWsclean(), name=imagename, size=imgsizepix, scale='5arcsec', \
weight='briggs 0.', niter=10000, no_update_model_required='', minuv_l=30, mgain=0.85, \
baseline_averaging='', parallel_deconvolution=512, \
auto_threshold=20, join_channels='', fit_spectral_pol=3, channels_out=9, deconvolution_channels=3)
# make mask
im = lib_img.Image(imagename + '-MFS-image.fits')
im.makeMask(threshisl=3)
logger.info('Cleaning w/ mask...')
lib_util.run_wsclean(s, 'wscleanB.log', MSs.getStrWsclean(), name=imagename, size=imgsizepix, scale='5arcsec', \
weight='briggs 0.', niter=100000, no_update_model_required='', minuv_l=30, mgain=0.85, \
baseline_averaging='', parallel_deconvolution=512, \
auto_threshold=0.1, fits_mask=im.maskname, join_channels='', fit_spectral_pol=3, channels_out=9, deconvolution_channels=3)
os.system(
'cat logs/wscleanA.log logs/wscleanB.log | grep "background noise"')
# make new mask
im.makeMask(threshisl=5)
logger.info("Done.")
def clean(p, MSs, res='normal', size=[1, 1], empty=False, imagereg=None):
"""
p = patch name
mss = list of mss to clean
size = in deg of the image
"""
# set pixscale and imsize
pixscale = MSs.resolution
if res == 'normal':
pixscale = float('%.1f' % (pixscale / 2.5))
elif res == 'high':
pixscale = float('%.1f' % (pixscale / 3.5))
elif res == 'low':
pass # no change
imsize = [
int(size[0] * 1.5 / (pixscale / 3600.)),
int(size[1] * 1.5 / (pixscale / 3600.))
] # add 50%
imsize[0] += imsize[0] % 2
imsize[1] += imsize[1] % 2
if imsize[0] < 256: imsize[0] = 256
if imsize[1] < 256: imsize[1] = 256
logger.debug('Image size: ' + str(imsize) + ' - Pixel scale: ' +
str(pixscale))
if res == 'normal':
weight = 'briggs -0.3'
maxuv_l = None
elif res == 'high':
weight = 'briggs -0.6'
maxuv_l = None
elif res == 'low':
weight = 'briggs 0'
maxuv_l = 3500
else:
logger.error('Wrong "res": %s.' % str(res))
sys.exit()
if empty:
logger.info('Cleaning empty (' + str(p) + ')...')
imagename = 'img/empty-' + str(p)
lib_util.run_wsclean(s,
'wscleanE-' + str(p) + '.log',
MSs.getStrWsclean(),
name=imagename,
data_column='SUBTRACTED_DATA',
size=imsize,
scale=str(pixscale) + 'arcsec',
weight=weight,
niter=0,
no_update_model_required='',
minuv_l=30,
mgain=0,
baseline_averaging='')
else:
# clean 1
logger.info('Cleaning (' + str(p) + ')...')
imagename = 'img/extract-' + str(p)
lib_util.run_wsclean(s,
'wscleanA-' + str(p) + '.log',
MSs.getStrWsclean(),
name=imagename,
size=imsize,
scale=str(pixscale) + 'arcsec',
weight=weight,
niter=10000,
no_update_model_required='',
minuv_l=30,
maxuv_l=maxuv_l,
mgain=0.85,
baseline_averaging='',
parallel_deconvolution=512,
auto_threshold=5,
join_channels='',
fit_spectral_pol=3,
channels_out=ch_out,
deconvolution_channels=3)
# make mask
im = lib_img.Image(imagename + '-MFS-image.fits', userReg=userReg)
try:
im.makeMask(threshpix=10, rmsbox=(70, 5))
except:
logger.warning('Fail to create mask for %s.' % imagename +
'-MFS-image.fits')
return
if imagereg is not None:
lib_img.blank_image_reg(im.maskname,
imagereg,
inverse=True,
blankval=0.)
# clean 2
# TODO: add deconvolution_channels when bug fixed
logger.info('Cleaning w/ mask (' + str(p) + ')...')
imagenameM = 'img/extractM-' + str(p)
lib_util.run_wsclean(
s,
'wscleanB-' + str(p) + '.log',
MSs.getStrWsclean(),
name=imagenameM,
do_predict=True,
size=imsize,
scale=str(pixscale) + 'arcsec',
weight=weight,
niter=100000,
no_update_model_required='',
minuv_l=30,
maxuv_l=maxuv_l,
reuse_psf=imagename,
reuse_dirty=imagename,
mgain=0.85,
multiscale='',
baseline_averaging='',
parallel_deconvolution=512,
auto_threshold=0.7,
auto_mask=1.5,
fits_mask=im.maskname,
join_channels='',
fit_spectral_pol=3,
channels_out=ch_out) # , deconvolution_channels=3)
os.system('cat logs/wscleanB-' + str(p) +
'.log | grep "background noise"')
os.system(
'cp ddcal/c01/solutions/interp.h5 extract/init/') # copy fnal dde sol
lib_util.check_rm('mss-extract')
if not os.path.exists('mss-extract'):
logger.info('Copy MS...')
os.system('cp -r mss-avg mss-extract')
# region must be a list of ds9 circles and polygons (other shapes can be implemented in lib_util.Rgion_helper()
target_reg = lib_util.Region_helper(target_reg_file)
center = target_reg.get_center() # center of the extract region
MSs = lib_ms.AllMSs(glob.glob('mss-extract/*MS'), s)
ch_out = MSs.getChout(4e6) # chout from dd-serial
fwhm = MSs.getListObj()[0].getFWHM(freq='mid')
phase_center = MSs.getListObj()[0].getPhaseCentre()
# read image, h5parm, make mask
wideDD_image = lib_img.Image('extract/init/wideDD-c01.app.restored.fits')
dde_h5parm = 'extract/init/interp.h5'
# make mask for subtraction
mask_ddcal = wideDD_image.imagename.replace(
'.fits', '_mask-ddcal.fits') # this is used to find calibrators
wideDD_image.makeMask(threshpix=5,
atrous_do=True,
maskname=mask_ddcal,
write_srl=True,
write_ds9=True)
def clean(p, MSs, res='normal', size=[1, 1], empty=False, imagereg=None):
"""
p = patch name
mss = list of mss to clean
lofar_system = 'hba'
flag_steps = "[ears, ant, uvmin, elev, count]"
########################################################
# flag bad stations, and low-elev
logger.info('Flagging...')
MSs.run('DPPP '+parset_dir+'/DPPP-flag.parset msin=$pathMS msout=. steps=\"'+flag_steps+'\" ant.baseline=\"'+bl2flag+'\"', \
log='$nameMS_flag.log', commandType='DPPP')
if lofar_system == 'hba':
model_dir = '/home/fdg/scripts/model/AteamHBA/' + patch
else:
model_dir = '/home/fdg/scripts/model/AteamLBA/' + patch
if os.path.exists(model_dir + '/img-MFS-model.fits'):
im = lib_img.Image(model_dir + '/img-MFS-image.fits')
im.rescaleModel(f)
n = len(glob.glob(model_dir + '/img-[0-9]*-model.fits'))
logger.info('Predict (wsclean: %s - chan: %i)...' % (model_dir, n))
s.add('wsclean -predict -name '+model_dir+'/img -j '+str(s.max_processors)+' -channels-out '+str(n)+' '+MSs.getStrWsclean(), \
log='wscleanPRE-init.log', commandType='wsclean', processors='max')
s.run(check=True)
else:
logger.info('Predict (DPPP)...')
MSs.run('DPPP ' + parset_dir +
'/DPPP-predict.parset msin=$pathMS pre.sourcedb=' + skymodel +
' pre.sources=' + patch,
log='$nameMS_pre.log',
commandType='DPPP')
# TESTTESTTEST
if imgsizepix % 2 != 0: imgsizepix += 1 # prevent odd img sizes
#MSs.getListObj()[0].makeBeamReg('ddcal/beam.reg', freq='mid')
#beamReg = 'ddcal/beam.reg'
logger.info('Add columns...')
MSs.run('addcol2ms.py -m $pathMS -c CORRECTED_DATA,SUBTRACTED_DATA -i DATA',
log='$nameMS_addcol.log',
commandType='python')
MSs.run('addcol2ms.py -m $pathMS -c FLAG_BKP -i FLAG',
log='$nameMS_addcol.log',
commandType='python')
##############################################################
# setup initial model
os.system('cp self/images/wideM-1* ddcal/init/')
full_image = lib_img.Image('ddcal/init/wideM-1-MFS-image.fits',
userReg=userReg)
for cmaj in range(maxIter):
logger.info('Starting major cycle: %i' % cmaj)
# cycle specific variables
picklefile = 'ddcal/directions-c%02i.pickle' % cmaj
interp_h5parm = 'ddcal/c%02i/solutions/interp.h5' % cmaj
#aterm_config_file = 'ddcal/c%02i/aterm/aterm.config' % cmaj
mask_ddcal = full_image.imagename.replace(
'.fits', '_mask-ddcal.fits') # this is used to find calibrators
if not os.path.exists('ddcal/c%02i' % cmaj):
os.makedirs('ddcal/c%02i' % cmaj)
for subdir in ['plots', 'images', 'solutions', 'skymodels']:
if not os.path.exists('ddcal/c%02i/%s' % (cmaj, subdir)):
target_reg = 'target.reg'
#######################################################
from LiLF import lib_ms, lib_img, lib_util, lib_log, lib_dd
logger_obj = lib_log.Logger('pipeline-facet_self.logger')
logger = lib_log.logger
s = lib_util.Scheduler(log_dir=logger_obj.log_dir,
dry=False) #, maxThreads = 4)
w = lib_util.Walker('pipeline-facet-self.walker')
# parse parset
parset = lib_util.getParset()
parset_dir = parset.get('LOFAR_facet_self', 'parset_dir')
maxniter = parset.getint('LOFAR_facet_self', 'maxniter')
userReg = parset.get('model', 'userReg')
mosaic_image = lib_img.Image('ddcal/images/c%02i/mos-MFS-image.fits' %
lastcycle)
############################
if w.todo('cleaning'):
logger.info('Cleaning...')
lib_util.check_rm('plot*')
lib_util.check_rm('cal*h5')
lib_util.check_rm('img')
lib_util.check_rm('facet')
os.makedirs('img')
os.makedirs('facet')
lib_util.check_rm('mss-facet')
if not os.path.exists('mss-facet'): os.system('cp -r mss-dd mss-facet')
w.done('cleaning')
Ss = lib_ms.AllMSs(glob.glob('mss-dd/TC*[0-9].MS'), s)
logger.info('Add columns...')
MSs.run('addcol2ms.py -m $pathMS -c CORRECTED_DATA,SUBTRACTED_DATA',
log='$nameMS_addcol.log',
commandType='python')
##############################################################
logger.info('BL-based smoothing...')
MSs.run('BLsmooth.py -f 1.0 -r -i DATA -o SMOOTHED_DATA $pathMS',
log='$nameMS_smooth.log',
commandType='python')
# setup initial model
mosaic_image = lib_img.Image(sorted(
glob.glob('self/images/wide-[0-9]-MFS-image.fits'))[-1],
userReg=userReg)
mosaic_image.selectCC()
rms_noise_pre = np.inf
for c in xrange(maxniter):
logger.info('Starting cycle: %i' % c)
lib_util.check_rm('img')
os.makedirs('img')
os.makedirs('ddcal/images/c%02i/regions' % c)
mask = 'ddcal/masks/facets%02i.fits' % c
### group into patches of similar flux
lsm = lsmtool.load(mosaic_image.skymodel_cut)
lsm.group('tessellate',
s.run(check=True)
# clean mask clean (cut at 5k lambda)
logger.info('Cleaning (cycle: ' + str(c) + ')...')
imagename = 'img/wide-' + str(c)
s.add('wsclean -reorder -temp-dir '+ temp_dir +' -name ' + imagename + ' -size ' + str(size) + ' ' + str(size) + ' -j '+str(s.max_processors)+' \
-scale 10arcsec -weight briggs 0. -niter 100000 -update-model-required -minuv-l 30 -maxuv-l 5000 -mgain 0.85 -clean-border 1 \
-multiscale -multiscale-scale-bias 0.5 -multiscale-scales 0,4,16 \
-auto-threshold 20 \
-use-idg \
-join-channels -fit-spectral-pol 2 -channels-out 10 ' +MSs.getStrWsclean(), \
log='wsclean-c'+str(c)+'.log', commandType='wsclean', processors='max')
s.run(check=True)
# make mask
im = lib_img.Image(imagename + '-MFS-image.fits', userReg=userReg)
im.makeMask(threshisl=3)
logger.info('Cleaning w/ mask (cycle: ' + str(c) + ')...')
s.add('wsclean -continue -reorder -temp-dir '+ temp_dir +' -name ' + imagename + ' -size ' + str(size) + ' ' + str(size) + ' -j '+str(s.max_processors)+' \
-scale 10arcsec -weight briggs 0. -niter 200000 -update-model-required -minuv-l 30 -maxuv-l 5000 -mgain 0.85 -clean-border 1 \
-multiscale -multiscale-scale-bias 0.5 -multiscale-scales 0,4,16 \
-auto-threshold 1 -fits-mask ' +im.maskname+' \
-use-idg \
-join-channels -fit-spectral-pol 2 -channels-out 10 -save-source-list ' +MSs.getStrWsclean(), \
log='wscleanM-c'+str(c)+'.log', commandType='wsclean', processors='max')
s.run(check=True)
os.system('cat logs/wscleanM-c' + str(c) +
'.log | grep "background noise"')
im = lib_img.Image(imagename + '-MFS-image.fits',
def clean(p, MSs, size, res='normal', apply_beam=False):
"""
p = patch name
mss = list of mss to clean
size = in deg of the image
"""
# set pixscale and imsize
pixscale = MSs.getListObj()[0].getResolution()
if res == 'normal':
pixscale = float('%.1f' % (pixscale / 2.))
elif res == 'high':
pixscale = float('%.1f' % (pixscale / 4.))
elif res == 'low':
pass # no change
imsize = [0, 0]
imsize[0] = int(size[0] * 1.05 / (pixscale / 3600.)) # add 5%
imsize[1] = int(size[1] * 1.05 / (pixscale / 3600.)) # add 5%
imsize[0] += imsize[0] % 2
imsize[1] += imsize[1] % 2
if imsize[0] < 64: imsize[0] == 64
if imsize[1] < 64: imsize[1] == 64
logger.debug('Image size: ' + str(imsize) + ' - Pixel scale: ' +
str(pixscale))
if res == 'normal':
weight = 'briggs 0'
maxuv_l = None
elif res == 'high':
weight = 'briggs -0.7'
maxuv_l = None
elif res == 'low':
weight = 'briggs 0'
maxuv_l = 3500
# clean 1
logger.info('Cleaning (' + str(p) + ')...')
imagename = 'img/ddcal-' + str(p)
lib_util.run_wsclean(s, 'wscleanA-'+str(p)+'.log', MSs.getStrWsclean(), name=imagename, size=imsize, scale=str(pixscale)+'arcsec', \
weight=weight, niter=10000, no_update_model_required='', minuv_l=30, maxuv_l=maxuv_l, mgain=0.8, \
baseline_averaging=5, parallel_deconvolution=256, auto_threshold=3, \
join_channels='', fit_spectral_pol=3, channels_out=9, deconvolution_channels=3)
# make mask
im = lib_img.Image(imagename + '-MFS-image.fits', userReg=userReg)
im.makeMask(threshisl=4)
# clean 2
# TODO: add -parallel-deconvolution when source lists can be saved (https://sourceforge.net/p/wsclean/tickets/141/)
logger.info('Cleaning w/ mask (' + str(p) + ')...')
imagename = 'img/ddcalM-' + str(p)
if apply_beam:
lib_util.run_wsclean(s, 'wscleanB-'+str(p)+'.log', MSs.getStrWsclean(), name=imagename, save_source_list='', size=imsize, scale=str(pixscale)+'arcsec', \
weight=weight, niter=100000, no_update_model_required='', minuv_l=30, maxuv_l=maxuv_l, mgain=0.8, \
use_idg='', grid_with_beam='', use_differential_lofar_beam='', beam_aterm_update=400, \
multiscale='', \
auto_threshold=0.5, fits_mask=im.maskname, \
join_channels='', fit_spectral_pol=3, channels_out=9, deconvolution_channels=3)
logger.info('Cleaning V (' + str(p) + ')...')
imagename = 'img/ddcalV-' + str(p)
lib_util.run_wsclean(s, 'wscleanV-'+str(p)+'.log', MSs.getStrWsclean(), name=imagename, size=imgsize, scale=str(pixscale)+'srcsec', pol='V', \
weight='briggs 0.', niter=1000, no_update_model_required='', minuv_l=30, maxuv_l=5000, \
baseline_averaging=5)
else:
lib_util.run_wsclean(s, 'wscleanB-'+str(p)+'.log', MSs.getStrWsclean(), name=imagename, size=imsize, save_source_list='', scale=str(pixscale)+'arcsec', \
weight=weight, niter=50000, no_update_model_required='', minuv_l=30, maxuv_l=maxuv_l, mgain=0.85, \
multiscale='', \
auto_threshold=0.5, fits_mask=im.maskname, \
baseline_averaging=5, join_channels='', fit_spectral_pol=3, channels_out=9, deconvolution_channels=3)
os.system('cat logs/wscleanA-' + str(p) + '.log logs/wscleanB-' + str(p) +
'.log | grep "background noise"')
logger.info('Sub model...')
MSs.run(
'taql "update $pathMS set CORRECTED_DATA = CORRECTED_DATA - MODEL_DATA"',
log='$nameMS_taql1.log',
commandType='general')
#logger.info('Copy MODEL_DATA...')
#MSs.run('taql "update $pathMS set MODEL_DATA_HIGHRES = MODEL_DATA"', log='$nameMS_taql2.log', commandType='general')
logger.info('Cleaning sub (cycle %i)...' % c)
imagename = 'img/imgsub-c' + str(c)
s.add('wsclean -reorder -temp-dir '+ temp_dir +' -name ' + imagename + ' -size 1000 1000 -j '+str(s.max_processors)+' \
-scale 15arcsec -weight briggs -1.0 -taper-gaussian 80arcsec -niter 10000 -no-update-model-required -minuv-l 30 -mgain 0.85 -clean-border 1 \
-multiscale -multiscale-scales 0,4,8,16 \
-use-idg \
-join-channels -fit-spectral-pol 2 -channels-out 15 ' +MSs.getStrWsclean(), \
log='wscleanSUB-c'+str(c)+'.log', commandType='wsclean', processors='max')
s.run(check=True)
#logger.info('Combining MODEL_DATA_HIGHRES and MODEL_DATA...')
#MSs.run('taql "update $pathMS set MODEL_DATA = MODEL_DATA_HIGHRES + MODEL_DATA"', log='$nameMS_taql3.log', commandType='general')
imagename = 'img/img-c' + str(c)
im = lib_img.Image(imagename)
im.rescaleModel(f)
logger.info('Predict (wsclean)...')
s.add('wsclean -predict -name '+imagename+' -j '+str(s.max_processors)+' -channelsout 15 '+MSs.getStrWsclean(), \
log='wscleanPRE-c'+str(c)+'.log', commandType='wsclean', processors='max')
s.run(check=True)
logger.info("Done.")
