def selectCC(self, checkBeam=True, keepInBeam=True, maskname=None):
"""
remove cc from a skymodel according to masks
checkBeam: remove according to beam (see keepInBeam)
keepInBeam: if beamReg is present and is True: remove sources outside beam
if beamReg is present and is False: remove source inside beam
maskname: a possible mask, otherwise try to use standard
"""
if maskname is None: maskname = self.maskname
if not os.path.exists(maskname):
raise("Missing mask in selectCC: %s." % maskname)
if checkBeam:
if self.beamReg is None:
raise('Missing beam in selectCC.')
logger.info('Predict (apply beam reg %s)...' % self.beamReg)
blank_image_reg(maskname, self.beamReg, inverse=keepInBeam, blankval=0) # if keep_in_beam set to 0 everything outside beam.reg
# apply mask
logger.info('%s: Apply mask (%s) on skymodel...' % (self.imagename,maskname))
lsm = lsmtool.load(self.skymodel)
lsm.select('%s == True' % maskname)
lsm.group('single') # group to 1 patch
lsm.write(self.skymodel_cut, format = 'makesourcedb', clobber=True)
del lsm
# convert from txt to blob
logger.info('%s: Make skydb...' % self.imagename)
lib_util.check_rm(self.skydb)
os.system('makesourcedb outtype="blob" format="<" in="'+self.skymodel_cut+'" out="'+self.skydb+'"')
def selectCC(self, keepInBeam=True):
"""
remove cc from a skymodel according to masks
keepInBeam: if beamReg is present and is True: remove sources outside beam
if beamReg is present and is False: remove source inside beam
"""
self.makeMask()
if self.facetReg is not None:
logger.info('Predict (apply facet reg %s)...' % self.facetReg)
blank_image_reg(self.maskname, self.facetReg, inverse=True, blankval=0) # set to 0 pixels outside facet mask
if self.beamReg is not None:
logger.info('Predict (apply beam reg %s)...' % self.beamReg)
blank_image_reg(self.maskname, self.beamReg, inverse=keepInBeam, blankval=0) # if keep_in_beam set to 0 everything outside beam.reg
# apply mask
logger.info('%s: Apply mask on skymodel...' % self.imagename)
lsm = lsmtool.load(self.skymodel)
lsm.select('%s == True' % self.maskname)
lsm.group('single') # group to 1 patch
lsm.write(self.skymodel_cut, format = 'makesourcedb', clobber=True)
del lsm
# convert from txt to blob
logger.info('%s: Make skydb...' % self.imagename)
lib_util.check_rm(self.skydb)
os.system('makesourcedb outtype="blob" format="<" in="'+self.skymodel_cut+'" out="'+self.skydb+'"')
def makeBeamReg(self, outfile, pb_cut=None, to_null=False, freq='mid'):
"""
Create a ds9 region of the beam
outfile : str
output file
pb_cut : float, optional
diameter of the beam
to_null : bool, optional
arrive to the first null, not the FWHM
freq: min,max,med
which frequency to use to estimate the beam size
"""
logger.debug('Making PB region: '+outfile)
ra, dec = self.getPhaseCentre()
if pb_cut is None:
radius = self.getFWHM(freq=freq)/2.
else:
radius = pb_cut/2.
if to_null: radius *= 2 # rough estimation
s = Shape('circle', None)
s.coord_format = 'fk5'
s.coord_list = [ ra, dec, radius ] # ra, dec, radius
s.coord_format = 'fk5'
s.attr = ([], {'width': '2', 'point': 'cross',
'font': '"helvetica 16 normal roman"'})
s.comment = 'color=red text="beam"'
regions = pyregion.ShapeList([s])
lib_util.check_rm(outfile)
regions.write(outfile)
def clean(p, MSs, size=2., apply_beam=False):
"""
p = patch name
mss = list of mss to clean
size = in deg of the image
"""
# set pixscale and imsize
pixscale = MSs.getListObj()[0].getResolution() / 3.
imsize = int(size / (pixscale / 3600.))
if imsize < 512:
imsize = 512
if imsize % 2 == 1: imsize += 1 # make even
logger.debug('Image size: ' + str(imsize) + ' - Pixel scale: ' +
str(pixscale))
if apply_beam:
idg_parms = '-use-idg -grid-with-beam -use-differential-lofar-beam -beam-aterm-update 400'
else:
idg_params = '-use-idg'
# clean 1
logger.info('Cleaning (' + str(p) + ')...')
imagename = 'img/ddcal-' + str(p)
s.add('wsclean -reorder -temp-dir /dev/shm -name ' + imagename + ' -size '+str(imsize)+' '+str(imsize)+' -j '+str(s.max_processors)+' \
-scale ' +str(pixscale)+'arcsec -weight briggs -0.5 -niter 100000 -no-update-model-required -minuv-l 30 -mgain 0.85 -clean-border 1 \
-auto-threshold 20 ' +idg_parms+' \
-join-channels -fit-spectral-pol 2 -channels-out 10 -save-source-list ' +MSs.getStrWsclean(), \
log='wsclean-'+str(p)+'.log', commandType='wsclean', processors='max')
s.run(check=True)
# make mask
im = lib_img.Image(imagename + '-MFS-image.fits', userReg=userReg)
os.system('mv %s %s' %
(im.skymodel, im.skymodel + '-first')) # copy the source list
im.makeMask(threshisl=3)
# clean 2
#-multiscale -multiscale-scale-bias 0.5 \
#-auto-mask 3 -rms-background-window 40 -rms-background-method rms-with-min \
logger.info('Cleaning w/ mask (' + str(p) + ')...')
s.add('wsclean -continue -reorder -temp-dir /dev/shm -name ' + imagename + ' -size '+str(imsize)+' '+str(imsize)+' -j '+str(s.max_processors)+' \
-scale ' +str(pixscale)+'arcsec -weight briggs -0.5 -niter 1000000 -no-update-model-required -minuv-l 30 -mgain 0.85 -clean-border 1 \
-auto-threshold 0.1 -fits-mask ' +im.maskname+' '+idg_parms+' \
-join-channels -fit-spectral-pol 2 -channels-out 10 -save-source-list ' +MSs.getStrWsclean(), \
log='wscleanM-'+str(p)+'.log', commandType='wsclean', processors='max')
s.run(check=True)
os.system('cat logs/wscleanM-' + str(p) + '.log | grep "background noise"')
os.system('grep -v \'^Format\' %s >> %s' %
(im.skymodel + '-first', im.skymodel)) # merge the source lists
lib_util.check_rm(im.skymodel + '-first')
return imagename
def move(self, pathMSNew, overwrite=False, keepOrig=False):
"""
Move (or rename) the MS to another locus in the file system.
"""
logger.debug('Move: '+self.pathMS+' -> '+pathMSNew)
if overwrite == True:
lib_util.check_rm(pathMSNew)
if not os.path.exists(pathMSNew):
if keepOrig:
shutil.copytree(self.pathMS, pathMSNew)
else:
shutil.move(self.pathMS, pathMSNew)
self.setPathVariables(pathMSNew)
def cut_skymodel(skymodel_in, skymodel_out, d, do_skydb=True, do_regions=False):
"""
Load full skymodel and extract sources in the square around the calibrator of the given size
"""
lsm = lsmtool.load(skymodel_in)
# select all sources within a sqare of patch size
lsm.select('Ra > %f' % (d.position[0]-(d.size/2)/np.cos(d.position[1]* np.pi / 180.)))
lsm.select('Ra < %f' % (d.position[0]+(d.size/2)/np.cos(d.position[1]* np.pi / 180.)))
lsm.select('Dec > %f' % (d.position[1]-d.size/2))
lsm.select('Dec < %f' % (d.position[1]+d.size/2))
if do_regions: lsm.write('ddcal/masks/regions-c%02i/%s.reg' % (cmaj,d.name), format='ds9', clobber=True)
lsm.write(dir_skymodel, format='makesourcedb', clobber=True)
lib_util.check_rm(dir_skydb)
s.add('makesourcedb outtype="blob" format="<" in="%s" out="%s"' % (dir_skymodel, dir_skydb), log='makesourcedb_cl.log', commandType='general' )
s.run(check=True)
def set_region(self, loc):
"""
Creates a ds9 regionfile that covers the DD-cal model
"""
assert self.size is not None and self.position is not None # we need this to be already set
self.region_file = loc+'/'+self.name+'.reg'
s = Shape('circle', None)
s.coord_format = 'fk5'
s.coord_list = [ self.position[0], self.position[1], self.size/2. ] # ra, dec, radius
s.coord_format = 'fk5'
s.attr = ([], {'width': '2', 'point': 'cross',
'font': '"helvetica 16 normal roman"'})
s.comment = 'color=red text="%s"' % self.name
regions = pyregion.ShapeList([s])
lib_util.check_rm(self.region_file)
regions.write(self.region_file)
sys.exit(1)
if avg_factor_f < 1 or keep_IS: avg_factor_f = 1
avg_factor_t = int(np.round(4 / timeint)) # to 4 sec
if avg_factor_t < 1 or keep_IS: avg_factor_t = 1
MSout = getName(MS.pathMS)
flog.write(MS.nameMS + '.MS\n')
if avg_factor_f != 1 or avg_factor_t != 1:
logger.info(
'%s->%s: Average in freq (factor of %i) and time (factor of %i)...'
% (MS.nameMS, MSout, avg_factor_f, avg_factor_t))
if keep_IS:
s.add('DPPP '+parset_dir+'/DPPP-avg.parset msin='+MS.pathMS+' msout='+MSout+' msin.datacolumn=DATA \
avg.timestep=' +str(avg_factor_t)+' avg.freqstep='+str(avg_factor_f), \
log=MS.nameMS+'_avg.log', commandType='DPPP')
else:
s.add('DPPP '+parset_dir+'/DPPP-avg.parset msin='+MS.pathMS+' msout='+MSout+' msin.datacolumn=DATA \
msin.baseline="[CR]S*&" \
avg.timestep=' +str(avg_factor_t)+' avg.freqstep='+str(avg_factor_f), \
log=MS.nameMS+'_avg.log', commandType='DPPP')
s.run(check=True,
maxThreads=1) # limit threads to prevent I/O isssues
lib_util.check_rm(MS.pathMS)
else:
logger.info('%s->%s: Move data - no averaging...' %
(MS.nameMS, MSout))
MS.move(MSout)
logger.info("Done.")
"""
for cal_dir in sorted(glob.glob('../../cals/*/*')):
cal_timestamps = set(
[ms.split('_')[1][1:] for ms in glob.glob(cal_dir + '/cals/*MS')])
if timestamp in cal_timestamps:
logger.info('Calibrator found: %s (t=%s)' % (cal_dir, timestamp))
return cal_dir + '/cals'
logger.error('Missing calibrator.')
sys.exit()
##########################################################
with w.if_todo('setup'):
logger.info('Cleaning...')
lib_util.check_rm('cal*h5')
lib_util.check_rm('plots*')
lib_util.check_rm('peel*')
lib_util.check_rm('img')
os.makedirs('img')
MSs = lib_ms.AllMSs(glob.glob(data_dir + '/*MS'), s, check_flags=False)
for timestamp in set(
[os.path.basename(ms).split('_')[1][1:] for ms in MSs.getListStr()]):
mss_toconcat = sorted(
glob.glob(data_dir + '/' + target + '_t' + timestamp + '_SB*.MS'))
MS_concat = target + '_t' + timestamp + '_concat.MS'
MS_concat_bkp = target + '_t' + timestamp + '_concat.MS-bkp'
if os.path.exists(MS_concat_bkp):
logger.info('Restoring bkp data: %s...' % MS_concat_bkp)
cor.ph.correction=phaseOrig000 cor.amp.correction=amplitude000 cor.amp.updateweights=False',
log='$nameMS_corIONO.log',
commandType="DPPP")
logger.info('Subtract model...')
MSs.run(
'taql "update $pathMS set CORRECTED_DATA = CORRECTED_DATA - MODEL_DATA"',
log='$nameMS_taql2.log',
commandType='general')
logger.info('Flag...')
MSs.run('DPPP ' + parset_dir + '/DPPP-flag2.parset msin=$pathMS',
log='$nameMS_flag2.log',
commandType='DPPP')
lib_util.check_rm('img')
os.makedirs('img')
imgsizepix = MSs.getListObj()[0].getFWHM() * 3600 / 5
logger.info('Cleaning low-res...')
imagename = 'img/calLR'
lib_util.run_wsclean(s, 'wscleanLR.log', MSs.getStrWsclean(), name=imagename, size=imgsizepix/5, scale='60arcsec', \
weight='briggs 0.', taper_gaussian='240arcsec', niter=10000, no_update_model_required='', minuv_l=30, maxuv_l=2000, mgain=0.85, \
use_idg='', grid_with_beam='', use_differential_lofar_beam='', beam_aterm_update=400, \
parallel_deconvolution=512, \
auto_mask=10, auto_threshold=1, pol='IQUV', join_polarizations='', join_channels='', fit_spectral_pol=3, channels_out=9, deconvolution_channels=3)
logger.info('Cleaning normal...')
imagename = 'img/cal'
lib_util.run_wsclean(s, 'wscleanA.log', MSs.getStrWsclean(), name=imagename, size=imgsizepix, scale='5arcsec', \
ateams_todemix = []
for ateam in ateams:
sep = MSs.getListObj()[0].distBrightSource(ateam)
if sep < 4 or sep > 15:
logger.debug('No demix of %s (sep: %.0f deg)' % (ateam, sep))
else:
ateams_todemix.append(ateam)
logger.warning('Demix of %s (sep: %.0f deg)' % (ateam, sep))
if len(ateams_todemix) < 0:
logger.info('Notingh to demix.')
sys.exit()
if os.path.exists('mss-predemix'):
logger.warning('Reset mss...')
lib_util.check_rm('mss/*MS')
else:
logger.info('Move mss in mss-predemix...')
os.system('mv mss mss-predemix')
os.system('mkdir mss')
MSs = lib_ms.AllMSs(glob.glob('mss-predemix/TC*[0-9].MS'), s)
if include_target:
if not os.path.exists('demix_combined.skymodel'):
logger.info('Including target...')
fwhm = MSs.getListObj()[0].getFWHM(freq='min')
phasecentre = MSs.getListObj()[0].getPhaseCentre()
radeg = phasecentre[0]
decdeg = phasecentre[1]
# get model the size of the image (radius=fwhm/2)
# parse parset
parset = lib_util.getParset()
parset_dir = parset.get('LOFAR_extract', 'parset_dir')
maxniter = parset.getint('LOFAR_extract', 'maxniter')
target_reg_file = parset.get('LOFAR_extract',
'extractRegion') # default 'target.reg'
phSolMode = parset.get('LOFAR_extract', 'phSolMode') # default: tecandphase
if phSolMode not in ['tecandphase', 'phase']:
logger.error('phSolMode {} not supported. Choose tecandphase, phase.')
sys.exit()
userReg = parset.get('model', 'userReg')
############################
with w.if_todo('cleaning'):
logger.info('Cleaning...')
lib_util.check_rm('extract')
lib_util.check_rm('img')
os.makedirs('img')
os.makedirs('extract/init')
os.system('cp ddcal/c01/images/wideDD-c01.app.restored.fits extract/init/'
) # copy ddcal image
os.system('cp ddcal/c01/images/wideDD-c01.DicoModel extract/init/'
) # copy dico model
os.system(
'cp ddcal/c01/solutions/interp.h5 extract/init/') # copy fnal dde sol
lib_util.check_rm('mss-extract')
if not os.path.exists('mss-extract'):
logger.info('Copy MS...')
os.system('cp -r mss-avg mss-extract')
# region must be a list of ds9 circles and polygons (other shapes can be implemented in lib_util.Rgion_helper()
import numpy as np
###############################################
from LiLF import lib_ms, lib_util, lib_log
logger_obj = lib_log.Logger('pipeline-demix.logger')
logger = lib_log.logger
s = lib_util.Scheduler(log_dir=logger_obj.log_dir, dry=False)
# parse parset
parset = lib_util.getParset()
parset_dir = parset.get('LOFAR_demix', 'parset_dir')
data_dir = parset.get('LOFAR_demix', 'data_dir')
skydb = parset.get('LOFAR_demix', 'demix_model')
##############################################
# Demix
MSs = lib_ms.AllMSs(glob.glob(data_dir + '/*MS'), s)
for MS in MSs.getListStr():
lib_util.check_rm(os.path.basename(MS) + '_' + os.path.basename(skydb))
print(('cp -r ' + skydb + ' ' + os.path.basename(MS) + '_' +
os.path.basename(skydb)))
os.system('cp -r ' + skydb + ' ' + os.path.basename(MS) + '_' +
os.path.basename(skydb))
logger.info('Demixing...')
MSs.run('DPPP '+parset_dir+'/DPPP_demix.parset msin=$pathMS msout=$nameMS demixer.skymodel=$nameMS.MS_'+os.path.basename(skydb)+' demixer.instrumentmodel=$nameMS/instrument_demix', \
log='$nameMS_demix.log', commandType='DPPP')
logger.info("Done.")
"""
for cal_dir in sorted(glob.glob('../../cals/*/*')):
cal_timestamps = set(
[ms.split('_')[1][1:] for ms in glob.glob(cal_dir + '/cals/*MS')])
if timestamp in cal_timestamps:
logger.info('Calibrator found: %s (t=%s)' % (cal_dir, timestamp))
return cal_dir + '/cals'
logger.error('Missing calibrator.')
sys.exit()
##########################################################
if w.todo('setup'):
logger.info('Cleaning...')
lib_util.check_rm('cal*h5')
lib_util.check_rm('plots*')
lib_util.check_rm('img')
os.makedirs('img')
MSs = lib_ms.AllMSs(glob.glob(data_dir + '/*MS'), s, check_flags=False)
for timestamp in set(
[os.path.basename(ms).split('_')[1][1:] for ms in MSs.getListStr()]):
mss_toconcat = sorted(
glob.glob(data_dir + '/' + target + '_t' + timestamp + '_SB*.MS'))
MS_concat = target + '_t' + timestamp + '_concat.MS'
MS_concat_bkp = target + '_t' + timestamp + '_concat.MS-bkp'
if os.path.exists(MS_concat_bkp):
logger.info('Restoring bkp data...')
lib_util.check_rm(MS_concat)
# parse parset
parset = lib_util.getParset()
parset_dir = parset.get('LOFAR_timesplit', 'parset_dir')
data_dir = parset.get('LOFAR_timesplit', 'data_dir')
cal_dir = parset.get('LOFAR_timesplit', 'cal_dir')
ngroups = parset.getint('LOFAR_timesplit', 'ngroups')
initc = parset.getint(
'LOFAR_timesplit',
'initc') # initial tc num (useful for multiple observation of same target)
bl2flag = parset.get('flag', 'stations')
#################################################
# Clean
with w.if_todo('clean'):
logger.info('Cleaning...')
lib_util.check_rm('mss*')
### DONE
with w.if_todo('copy'):
MSs = lib_ms.AllMSs(glob.glob(data_dir + '/*MS'), s)
logger.info('Copy data...')
for MS in MSs.getListObj():
#if min(MS.getFreqs()) > 30.e6:
# overwrite=True to prevent updating the weights twice
MS.move(MS.nameMS + '.MS', keepOrig=True, overwrite=True)
### DONE
MSs = lib_ms.AllMSs(glob.glob('*MS'), s)
##################################################
logger_obj = lib_log.Logger('pipeline-self.logger')
logger = lib_log.logger
s = lib_util.Scheduler(log_dir=logger_obj.log_dir, dry=False)
w = lib_util.Walker('pipeline-self.walker')
parset = lib_util.getParset()
parset_dir = parset.get('LOFAR_self', 'parset_dir')
sourcedb = parset.get('model', 'sourcedb')
apparent = parset.getboolean('model', 'apparent')
userReg = parset.get('model', 'userReg')
#############################################################################
# Clear
if w.todo('cleaning'):
logger.info('Cleaning...')
lib_util.check_rm('img')
os.makedirs('img')
# here images, models, solutions for each group will be saved
lib_util.check_rm('self')
if not os.path.exists('self/images'): os.makedirs('self/images')
if not os.path.exists('self/solutions'): os.makedirs('self/solutions')
if not os.path.exists('self/plots'): os.makedirs('self/plots')
w.done('cleaning')
### DONE
MSs = lib_ms.AllMSs(glob.glob('mss/TC*[0-9].MS'), s)
try:
MSs.print_HAcov()
except:
else:
lib_util.run_wsclean(s, 'wscleanB-'+str(p)+'.log', MSs.getStrWsclean(), name=imagename, size=imsize, save_source_list='', scale=str(pixscale)+'arcsec', \
weight=weight, niter=100000, no_update_model_required='', minuv_l=30, maxuv_l=maxuv_l, mgain=0.85, \
multiscale='', multiscale_scales='0,10,20,40,80',
baseline_averaging='', parallel_deconvolution=512, local_rms='', auto_threshold=0.75, auto_mask=1.5, fits_mask=im.maskname, \
join_channels='', fit_spectral_pol=3, channels_out=9, deconvolution_channels=3)
os.system('cat logs/wscleanA-' + str(p) + '.log logs/wscleanB-' + str(p) +
'.log | grep "background noise"')
#############################################################
with w.if_todo('cleaning'):
logger.info('Cleaning...')
lib_util.check_rm('ddcal')
os.makedirs('ddcal/masks')
os.makedirs('ddcal/plots')
os.makedirs('ddcal/images')
os.makedirs('ddcal/solutions')
os.makedirs('ddcal/skymodels')
### DONE
############################################################
# use SUBTRACTED_DATA (no pre-correction - subtraction would not work) or CORRECTED_DATA (DIE iono correction)?
logger.info('Copy data...')
if not os.path.exists('mss-dd'):
os.makedirs('mss-dd')
MSs_self.run('DPPP '+parset_dir+'/DPPP-avg.parset msin=$pathMS msout=mss-dd/$nameMS.MS msin.datacolumn=CORRECTED_DATA avg.freqstep=1 avg.timestep=1', \
log='$nameMS_avg.log', commandType='DPPP')
parset = lib_util.getParset()
parset_dir = parset.get('LOFAR_dd-incr', 'parset_dir')
maxniter = parset.getint('LOFAR_dd', 'maxniter')
calFlux = parset.getfloat('LOFAR_dd', 'calFlux')
userReg = parset.get('model', 'userReg')
####################################################
MSs_self = lib_ms.AllMSs(glob.glob('mss/TC*[0-9].MS'), s)
# make beam
phasecentre = MSs_self.getListObj()[0].getPhaseCentre()
fwhm = MSs_self.getListObj()[0].getFWHM(freq='mid')
############################
logger.info('Cleaning...')
lib_util.check_rm('ddcal')
os.makedirs('ddcal/masks')
os.makedirs('ddcal/plots')
os.makedirs('ddcal/images')
os.makedirs('ddcal/solutions')
os.makedirs('ddcal/skymodels')
def clean(p, MSs, size, res='normal', apply_beam=False):
"""
p = patch name
mss = list of mss to clean
size = in deg of the image
"""
# set pixscale and imsize
pixscale = MSs.getListObj()[0].getResolution()
dry=False) #, maxThreads = 4)
w = lib_util.Walker('pipeline-facet-self.walker')
# parse parset
parset = lib_util.getParset()
parset_dir = parset.get('LOFAR_facet_self', 'parset_dir')
maxniter = parset.getint('LOFAR_facet_self', 'maxniter')
userReg = parset.get('model', 'userReg')
mosaic_image = lib_img.Image('ddcal/images/c%02i/mos-MFS-image.fits' %
lastcycle)
############################
if w.todo('cleaning'):
logger.info('Cleaning...')
lib_util.check_rm('plot*')
lib_util.check_rm('cal*h5')
lib_util.check_rm('img')
lib_util.check_rm('facet')
os.makedirs('img')
os.makedirs('facet')
lib_util.check_rm('mss-facet')
if not os.path.exists('mss-facet'): os.system('cp -r mss-dd mss-facet')
w.done('cleaning')
### DONE
MSs = lib_ms.AllMSs(glob.glob('mss-facet/*MS'), s)
def clean(p, MSs, size, res='normal', apply_beam=False):
########################################################
from LiLF import lib_ms, lib_img, lib_util, lib_log
logger_obj = lib_log.Logger('pipeline-self.logger')
logger = lib_log.logger
s = lib_util.Scheduler(log_dir=logger_obj.log_dir, dry=False)
parset = lib_util.getParset()
parset_dir = parset.get('LOFAR_self', 'parset_dir')
sourcedb = parset.get('model', 'sourcedb')
apparent = parset.getboolean('model', 'apparent')
userReg = parset.get('model', 'userReg')
#############################################################################
# Clear
logger.info('Cleaning...')
lib_util.check_rm('img')
os.makedirs('img')
# here images, models, solutions for each group will be saved
lib_util.check_rm('self')
if not os.path.exists('self/images'): os.makedirs('self/images')
if not os.path.exists('self/solutions'): os.makedirs('self/solutions')
if not os.path.exists('self/plots'): os.makedirs('self/plots')
lib_util.check_rm('img')
os.makedirs('img')
MSs = lib_ms.AllMSs(glob.glob('mss/TC*[0-9].MS'), s)
try:
MSs.plot_HAcov('HAcov.png')
except:
logger.error('Problem with HAcov, continue anyway.')
imagename = 'img/ddcalM-' + str(p)
lib_util.run_wsclean(s, 'wscleanB-'+str(p)+'.log', MSs.getStrWsclean(), name=imagename, do_predict=True, \
size=imsize, save_source_list='', scale=str(pixscale)+'arcsec', \
weight=weight, niter=100000, no_update_model_required='', minuv_l=30, maxuv_l=maxuv_l, mgain=0.85, \
multiscale='', multiscale_scale_bias=0.65, multiscale_scales='0,10,20,40,80',
baseline_averaging='', parallel_deconvolution=512, local_rms='', auto_threshold=0.75, auto_mask=1.5, fits_mask=im.maskname, \
join_channels='', fit_spectral_pol=3, channels_out=ch_out) #, deconvolution_channels=3)
os.system('cat logs/wscleanA-' + str(p) + '.log logs/wscleanB-' +
str(p) + '.log | grep "background noise"')
#############################################################
if w.todo('cleaning'):
logger.info('Cleaning...')
lib_util.check_rm('ddcal')
os.makedirs('ddcal/init')
lib_util.check_rm('img')
os.makedirs('img')
w.done('cleaning')
### DONE
# goes down to 8 seconds/4 chans
if not os.path.exists('mss-avg'):
MSs = lib_ms.AllMSs(glob.glob('mss/TC*[0-9].MS'), s)
timeint = MSs.getListObj()[0].getTimeInt()
avgtimeint = int(round(8 / timeint))
nchan_init = MSs.getListObj()[0].getNchan()
# avg (x8) sol (x6) - we need a multiple of 8x6=48, the largest that is <nchan
# survey after avg (x8): 60, final number of sol 10
logger = lib_log.logger
s = lib_util.Scheduler(log_dir=logger_obj.log_dir, dry=False)
parset = lib_util.getParset()
parset_dir = parset.get('uGMRT_self', 'parset_dir')
sourcedb = parset.get('model',
'sourcedb') # relative to tgts dir "./tgts/xxx/model.txt
userReg = parset.get('model',
'userReg') # relative to tgts dir "./tgts/xxx/region.ref"
MSs = lib_ms.AllMSs(glob.glob('mss/*.MS'), s)
############################################################################
# Clear
logger.info('Cleaning...')
lib_util.check_rm('img')
os.makedirs('img')
lib_util.check_rm('self')
os.makedirs('self/images')
os.makedirs('self/solutions')
os.makedirs('self/plots')
is_wideband = len(
MSs.getListObj()[0].getFreqs()) > 1000 # if > 1000 chans, it is wideband
# make beam
phasecentre = MSs.getListObj()[0].getPhaseCentre()
MSs.getListObj()[0].makeBeamReg('self/beam.reg')
beamReg = 'self/beam.reg'
#################################################################
local_rms='',
auto_threshold=0.75,
auto_mask=1.5,
fits_mask=im.maskname,
join_channels='',
fit_spectral_pol=3,
channels_out=ch_out) #, deconvolution_channels=3)
os.system('cat logs/wscleanB-' + str(p) +
'.log | grep "background noise"')
#############################################################
with w.if_todo('cleaning'):
logger.info('Cleaning...')
lib_util.check_rm('ddcal')
os.makedirs('ddcal/init')
lib_util.check_rm('img')
os.makedirs('img')
lib_util.check_rm('mss-avg')
### DONE
# goes down to 8 seconds/4 chans
if not os.path.exists('mss-avg'):
MSs = lib_ms.AllMSs(glob.glob('mss/TC*[0-9].MS'), s)
timeint = MSs.getListObj()[0].getTimeInt()
avgtimeint = int(round(8 / timeint)) # to 8 seconds
nchan_init = MSs.getListObj()[0].getNchan()
# chan: avg (x8) sol (x6) - we need a multiple of 8x6=48, the largest that is <nchan
# survey after avg (x8): 60, final number of sol 10
# pointed after avg (x8): 120, final number of sol 20
s = lib_util.Scheduler(log_dir = logger_obj.log_dir, dry = False)
# parse parset
parset = lib_util.getParset()
parset_dir = parset.get('LOFAR_timesplit','parset_dir')
data_dir = parset.get('LOFAR_timesplit','data_dir')
cal_dir = parset.get('LOFAR_timesplit','cal_dir')
ngroups = parset.getint('LOFAR_timesplit','ngroups')
initc = parset.getint('LOFAR_timesplit','initc') # initial tc num (useful for multiple observation of same target)
if 'LBAsurvey' in os.getcwd():
data_dir = '/home/fdg/lofar1/LBAsurvey/%s/%s' % (os.getcwd().split('/')[-2], os.getcwd().split('/')[-1])
#################################################
# Clean
logger.info('Cleaning...')
lib_util.check_rm('mss*')
MSs = lib_ms.AllMSs( glob.glob(data_dir+'/*MS'), s )
logger.info('Copy data...')
for MS in MSs.getListObj():
if min(MS.getFreqs()) > 30.e6:
# overwrite=True to prevent updating the weights twice
MS.move(MS.nameMS+'.MS', keepOrig=True, overwrite=True)
MSs = lib_ms.AllMSs( glob.glob('*MS'), s )
##################################################
# Find solutions to apply
if 'LBAsurvey' in os.getcwd():
#cal_dir = 'portal_lei:/disks/paradata/fdg/LBAsurvey/cal_'+os.getcwd().split('/')[-2]+'_3C*/'
for cal_dir in glob.glob('../3c*'):
########################################################
from LiLF import lib_ms, lib_img, lib_util, lib_log
logger_obj = lib_log.Logger('pipeline-ateam.logger')
logger = lib_log.logger
s = lib_util.Scheduler(log_dir=logger_obj.log_dir, dry=False)
# parse parset
parset = lib_util.getParset()
parset_dir = parset.get('LOFAR_ateam', 'parset_dir')
bl2flag = parset.get('flag', 'stations')
data_dir = '../tgts-bkp/'
##########################################################
logger.info('Cleaning...')
#lib_util.check_rm('cal*h5')
lib_util.check_rm('plots*')
lib_util.check_rm('img')
os.makedirs('img')
MSs = lib_ms.AllMSs(sorted(glob.glob(data_dir + '/*MS')), s)
# copy data (avg to 1ch/sb and 10 sec)
nchan = int(MSs.getListObj()[0].getNchan()) # average 4 chans
timeint = MSs.getListObj()[0].getTimeInt()
avg_time = int(np.rint(30. / timeint)) # average to 30 s
logger.info('Copy data...')
for obs in set([os.path.basename(ms).split('_')[0]
for ms in MSs.getListStr()]):
mss_toconcat = sorted(glob.glob(data_dir + '/' + obs + '*MS'))
w = lib_util.Walker('script-benchmark.walker')
parset = lib_util.getParset()
parset_dir = parset.get('LOFAR_self', 'parset_dir')
parser = argparse.ArgumentParser(description="Benchmark LOFAR software.")
parser.add_argument("msfiles",
type=str,
help="Input ms files, glob-like string")
parser.add_argument("--sourcedb",
default='',
help="If you want to provide a certain skymodel.")
args = parser.parse_args()
#############################################################################
# Clear
with w.if_todo('cleaning'):
lib_util.check_rm('bench')
if not os.path.exists('bench/img'): os.makedirs('bench/img')
logger.info('Cleaning...')
lib_util.check_rm('bench/img')
os.makedirs('bench/img')
lib_util.check_rm('bench/plots')
os.makedirs('bench/plots')
### DONE
print(args.msfiles)
MSs = lib_ms.AllMSs(glob.glob(args.msfiles), s)
sourcedb = args.sourcedb
ncpu = mp.cpu_count()
nms = len(MSs.getListObj())
logger.info(f'Found {ncpu} cpus and {nms} MS files')
userReg = parset.get('model', 'userReg')
uvrange = [0.03, 1e6]
image_robust = -0.15
ncluster = 5
####################################################
MSs_self = lib_ms.AllMSs(glob.glob('mss/TC*[0-9].MS'), s)
# make beam
phasecentre = MSs_self.getListObj()[0].getPhaseCentre()
fwhm = MSs_self.getListObj()[0].getFWHM(freq='mid')
nchan = MSs_self.getListObj()[0].getNchan()
############################
logger.info('Cleaning...')
lib_util.check_rm('ddfcal')
os.makedirs('ddfcal/masks')
os.makedirs('ddfcal/plots')
os.makedirs('ddfcal/images')
os.makedirs('ddfcal/solutions')
os.makedirs('ddfcal/cache')
lib_util.check_rm('img')
os.makedirs('img')
# Clear the shared memory
s.add('ClearSHM.py',
log='clearSHM.log',
commandType='singularity',
processors='max')
s.run(check=True)
