# Smoothing - ms:SUBTRACTED_DATA -> ms:SMOOTHED_DATA
logger.info('BL-based smoothing...')
MSs.run(
'BLsmooth.py -c 8 -r -i SUBTRACTED_DATA -o SMOOTHED_DATA $pathMS',
log='$nameMS_smooth-c' + str(c) + '.log',
commandType='python')
# Calibration - ms:SMOOTHED_DATA
logger.info('Gain calibration...')
MSs.run('DPPP '+parset_dir+'/DPPP-solG.parset msin=$pathMS \
sol.h5parm=$pathMS/cal-g-c' +str(c)+'.h5 sol.sourcedb='+skymodel_cl_skydb, \
log='$nameMS_solG-c'+str(c)+'.log', commandType='DPPP')
# Plot solutions
lib_util.run_losoto(s, 'g-c'+str(c), [ms+'/cal-g-c'+str(c)+'.h5' for ms in MSs.getListStr()], \
[parset_dir+'/losoto-amp.parset', parset_dir+'/losoto-plot-amp.parset', parset_dir+'/losoto-plot-ph.parset'])
os.system('mv plots-g-c' + str(c) + ' ddcal/plots')
os.system('mv cal-g-c' + str(c) + '.h5 ddcal/solutions')
### DONE
###########################################################
# use idg and A-term to correct the data, single image
if aterm_imaging:
#wsclean -mem 90.0 -scale 0.0004166666666666667 -aterm-config /beegfs/rafferty/Data/LOFAR/Screens/Factor_sim/pipelines/image_1/sector_3/chunk9.ms.make_aterm_config -multiscale-scales 0 -size 1500 1500 -deconvolution-channels 4 -fits-mask /beegfs/rafferty/Data/LOFAR/Screens/Factor_sim/pipelines/image_1/sector_3/chunk9.ms.premask -j 6 -auto-mask 3.6 -idg-mode hybrid -channels-out 12 -local-rms-window 50 -mgain 0.5 -minuv-l 80.0 -fit-spectral-pol 3 -maxuv-l 1000000.0 -weighting-rank-filter 3 -aterm-kernel-size 32 -temp-dir /tmp -name /beegfs/rafferty/Data/LOFAR/Screens/Factor_sim/pipelines/image_1/sector_3/chunk9.ms.image -padding 1.2 -pol I -multiscale-shape gaussian -auto-threshold 1.0 -local-rms-method rms-with-min -weight briggs -0.5 -niter 13635 -no-update-model-required -multiscale -fit-beam -reorder -save-source-list -local-rms -join-channels -use-idg -apply-primary-beam -nmiter 4
lib_util.run_wsclean(s, 'wscleanDD-c%02i.log' %c, MSs.getStrWsclean(), name='img/wideDD-c%02i' %c, save_source_list='', size=imsize, scale=str(pixscale)+'arcsec', \
weight=weight, niter=100000, no_update_model_required='', minuv_l=30, maxuv_l=maxuv_l, mgain=0.85, \
use_idg='', grid_with_beam='', use_differential_lofar_beam='', beam_aterm_update=400, \
multiscale='', multiscale_scales='0,10,20,40,80', \
parallel_deconvolution=512, local_rms='', auto_threshold=0.75, auto_mask=1.5, fits_mask=im.maskname, \
# Smooth data DATA -> SMOOTHED_DATA (BL-based smoothing)
logger.info('BL-smooth...')
MSs.run('BLsmooth.py -r -c 1 -n 8 -i DATA -o SMOOTHED_DATA $pathMS',
log='$nameMS_smooth1.log',
commandType='python',
maxThreads=8)
# Solve cal_SB.MS:SMOOTHED_DATA (only solve)
logger.info('Calibrating PA...')
MSs.run(
'DPPP ' + parset_dir +
'/DPPP-soldd.parset msin=$pathMS sol.h5parm=$pathMS/pa.h5 sol.mode=rotation+diagonal',
log='$nameMS_solPA.log',
commandType="DPPP")
lib_util.run_losoto(s, 'pa', [ms+'/pa.h5' for ms in MSs.getListStr()], \
[parset_dir+'/losoto-plot-ph.parset', parset_dir+'/losoto-plot-rot.parset', parset_dir+'/losoto-plot-amp.parset', parset_dir+'/losoto-pa.parset'])
# Pol align correction DATA -> CORRECTED_DATA
logger.info('Polalign correction...')
MSs.run(
'DPPP ' + parset_dir +
'/DPPP-cor.parset msin=$pathMS msin.datacolumn=DATA cor.parmdb=cal-pa.h5 cor.correction=polalign',
log='$nameMS_corPA.log',
commandType="DPPP")
w.done('cal_pa')
### DONE
########################################################
# 2: find FR
####################################################
# 1: Solving
with w.if_todo('calib-ph-c%02i' % c):
# solve G - group*_TC.MS:CORRECTED_DATA
logger.info('Solving fast...')
solint = next(solint_ph)
MSs.run(
'DPPP ' + parset_dir +
'/DPPP-solG.parset msin=$pathMS msin.datacolumn=DATA sol.h5parm=$pathMS/calGp.h5 sol.mode=scalarcomplexgain \
sol.solint=' + str(solint) + ' sol.smoothnessconstraint=5e6',
log='$nameMS_solGp-c' + str(c) + '.log',
commandType="DPPP")
lib_util.run_losoto(s, 'Gp-c' + str(c),
[ms + '/calGp.h5' for ms in MSs.getListStr()], [
parset_dir + '/losoto-clip-large.parset',
parset_dir + '/losoto-plot2d.parset',
parset_dir + '/losoto-plot.parset'
])
# Correct DATA -> CORRECTED_DATA
logger.info('Correction PH...')
MSs.run('DPPP ' + parset_dir + '/DPPP-cor.parset msin=$pathMS msin.datacolumn=DATA cor.parmdb=cal-Gp-c'+str(c)+'.h5 cor.correction=phase000', \
log='$nameMS_corPH-c'+str(c)+'.log', commandType='DPPP')
### DONE
if doamp:
with w.if_todo('calib-amp-c%02i' % c):
# solve G - group*_TC.MS:CORRECTED_DATA
logger.info('Solving slow...')
#sol.antennaconstraint=[[CSsuperLBA,CS001LBA,CS002LBA,CS003LBA,CS004LBA,CS005LBA,CS006LBA,CS007LBA,CS011LBA,CS013LBA,CS017LBA,CS021LBA,CS024LBA,CS026LBA,CS028LBA,CS030LBA,CS031LBA,CS032LBA,CS101LBA,CS103LBA,CS201LBA,CS301LBA,CS302LBA,CS401LBA,CS501LBA,RS106LBA,RS205LBA,RS208LBA,RS210LBA,RS305LBA,RS306LBA,RS307LBA,RS310LBA,RS406LBA,RS407LBA,RS409LBA,RS503LBA,RS508LBA,RS509LBA]] \
commandType='python')
####################################################
# 1: find PA and remove it
# Solve cal_SB.MS:DATA (only solve)
if not os.path.exists('cal-pa-c0.h5'):
logger.info('Solving PA...')
MSs.run(
'DPPP ' + parset_dir +
'/DPPP-soldd.parset msin=$pathMS msin.datacolumn=DATA sol.h5parm=$pathMS/pa.h5 sol.mode=rotation+diagonal \
sol.uvlambdarange=' + str(nouseblrange),
log='$nameMS_solPA.log',
commandType="DPPP")
lib_util.run_losoto(s, 'pa-c'+str(c), [ms+'/pa.h5' for ms in MSs.getListStr()], \
[parset_dir+'/losoto-plot-ph.parset', parset_dir+'/losoto-plot-rot.parset', parset_dir+'/losoto-plot-amp.parset', parset_dir+'/losoto-pa.parset'])
#################################################
# 2: find the FR and remve it
# Beam correction DATA -> CORRECTED_DATA
logger.info('PA correction...')
MSs.run('DPPP '+parset_dir+'/DPPP-cor.parset msin=$pathMS msin.datacolumn=DATA cor.parmdb=cal-pa-c0.h5 cor.correction=polalign', \
log='$nameMS_corPA2.log', commandType="DPPP")
# Beam correction CORRECTED_DATA -> CORRECTED_DATA
logger.info('Beam correction...')
MSs.run('DPPP ' + parset_dir + '/DPPP-beam.parset msin=$pathMS',
log='$nameMS_beam2.log',
commandType='DPPP')
solint_amp2 = next(iter_amp2_solint)
if phSolMode == 'phase':
logger.info('Phase calibration...')
with w.if_todo('cal-ph-c%02i' % c):
MSs.run(
'DPPP ' + parset_dir +
'/DPPP-solG.parset msin=$pathMS msin.datacolumn=SMOOTHED_DATA sol.h5parm=$pathMS/cal-ph.h5 \
sol.mode=scalarphase sol.solint=' + str(solint_ph) +
' sol.smoothnessconstraint=5e6 \
sol.antennaconstraint=[[CS001LBA,CS002LBA,CS003LBA,CS004LBA,CS005LBA,CS006LBA,CS007LBA,CS011LBA,CS013LBA,CS017LBA,CS021LBA,CS024LBA,CS026LBA,CS028LBA,CS030LBA,CS031LBA,CS032LBA,CS101LBA,CS103LBA,CS201LBA,CS301LBA,CS302LBA,CS401LBA,CS501LBA]]',
log='$nameMS_solGph-c%02i.log' % c,
commandType='DPPP')
lib_util.run_losoto(s,
'ph',
[ms + '/cal-ph.h5' for ms in MSs.getListStr()],
[parset_dir + '/losoto-plot1.parset'],
plots_dir='extract/plots-%s' % c)
os.system('mv cal-ph.h5 %s' % h5ph)
with w.if_todo('cor-ph-c%02i' % c):
# correct ph - ms:DATA -> ms:CORRECTED_DATA
logger.info('Correct ph...')
MSs.run(
'DPPP ' + parset_dir +
'/DPPP-correct.parset msin=$pathMS msin.datacolumn=DATA msout.datacolumn=CORRECTED_DATA \
cor.parmdb=' + h5ph + ' cor.correction=phase000',
log='$nameMS_correct-c%02i.log' % c,
commandType='DPPP')
else:
logger.info('Tecandphase calibration...')
dicomodel='%s.DicoModel' %
rootimg_old,
cache_dir='ddfcal/cache',
sols_dir='ddfcal/solutions',
clusterfile=clusterfile,
CovQ=0.02,
niterkf=6,
uvrange=uvrange,
wtuv=None,
robust=image_robust,
dt=dt,
NChanSols=12,
MergeSmooth=True)
# run losoto
lib_util.run_losoto(s, 'g-c'+str(c), glob.glob('ddfcal/solutions/TC*.MS/*npz'), \
[parset_dir+'/losoto-plot-amp.parset', parset_dir+'/losoto-plot-ph.parset'])
os.system('mv plots-g-c' + str(c) + ' ddfcal/plots')
os.system('mv cal-g-c' + str(c) + '.h5 ddfcal/solutions')
# image
if c <= 1: applysols = 'P'
else: applysols = 'P'
logger.info('Imaging...')
lib_ddfacet.ddf_image(s, 'ddf%02i.log' % c, MSs, rootimg,cleanmode='SSD',\
ddsols=solname,applysols=applysols,\
majorcycles=2,robust=image_robust,\
peakfactor=0.001,colname='DATA',\
automask=True,automask_threshold=10, cleanmask=maskname, \
normalization='BLBased',apply_weights=True,use_weightspectrum=False,uvrange=uvrange,
use_dicomodel=True,dicomodel_base=rootimg_old,
# Calibration - ms:SMOOTHED_DATA
# possible to put nchan=6 if less channels are needed in the h5parm (e.g. for IDG)
logger.info('Gain phase calibration (solint: %i)...' %
solint_ph)
MSs_dir.run(
'DPPP ' + parset_dir +
'/DPPP-solG.parset msin=$pathMS msin.datacolumn=SMOOTHED_DATA sol.h5parm=$pathMS/cal-ph.h5 \
sol.mode=scalarphase sol.solint=' + str(solint_ph) +
' sol.smoothnessconstraint=5e6 \
sol.antennaconstraint=[[CS001LBA,CS002LBA,CS003LBA,CS004LBA,CS005LBA,CS006LBA,CS007LBA,CS011LBA,CS013LBA,CS017LBA,CS021LBA,CS024LBA,CS026LBA,CS028LBA,CS030LBA,CS031LBA,CS032LBA,CS101LBA,CS103LBA,CS201LBA,CS301LBA,CS302LBA,CS401LBA,CS501LBA]]',
log='$nameMS_solGph-' + logstringcal + '.log',
commandType='DPPP')
lib_util.run_losoto(
s,
'ph', [ms + '/cal-ph.h5' for ms in MSs_dir.getListStr()],
[parset_dir + '/losoto-plot1.parset'],
plots_dir='ddcal/c%02i/plots/plots-%s' %
(cmaj, logstringcal))
os.system('mv cal-ph.h5 %s' % d.get_h5parm('ph'))
# correct ph - ms:DATA -> ms:CORRECTED_DATA
logger.info('Correct ph...')
MSs_dir.run(
'DPPP ' + parset_dir +
'/DPPP-correct.parset msin=$pathMS msin.datacolumn=DATA msout.datacolumn=CORRECTED_DATA \
cor.parmdb=' + d.get_h5parm('ph') +
' cor.correction=phase000',
log='$nameMS_correct-' + logstringcal + '.log',
commandType='DPPP')
if doamp:
#parset = lib_util.getParset()
parset_dir = '/home/baq1889/scripts/LiLF/parsets/LOFAR_dd'
#############################################################
MSs = lib_ms.AllMSs( glob.glob('*MS'), s, check_flags=False )
skymodel_voro_skydb = 'skymodel00_voro.skydb'
c=0
# Calibration - ms:SMOOTHED_DATA
logger.info('Calibrating...')
MSs.run('DPPP '+parset_dir+'/DPPP-solDD.parset msin=$pathMS ddecal.h5parm=$pathMS/cal-core.h5 \
ddecal.sourcedb='+skymodel_cl_skydb+' ddecal.solint=15 ddecal.nchan=30', \
log='$nameMS_solDD-core.log', commandType='DPPP')
lib_util.run_losoto(s, 'core', [MS+'/cal-core.h5' for MS in MSs.getListStr()], \
[parset_dir+'/losoto-core.parset'])
logger.info('re-calibration...')
# predict and corrupt each facet
logger.info('Reset MODEL_DATA...')
MSs.run('taql "update $pathMS set MODEL_DATA = 0"', log='$nameMS_taql-c'+str(c)+'.log', commandType='general')
for i, d in enumerate(directions):
# predict - ms:MODEL_DATA
logger.info('Patch '+d.name+': predict...')
MSs.run('DPPP '+parset_dir+'/DPPP-predict.parset msin=$pathMS msout.datacolumn=MODEL_DATA_DIR pre.sourcedb='+skymodel_voro_skydb+' pre.sources='+d.name, \
log='$nameMS_pre1-c'+str(c)+'-'+d.name+'.log', commandType='DPPP')
# corrupt - ms:MODEL_DATA -> ms:MODEL_DATA
logger.info('Patch '+d.name+': corrupt...')
MSs.run('DPPP '+parset_dir+'/DPPP-corrupt.parset msin=$pathMS msin.datacolumn=MODEL_DATA_DIR msout.datacolumn=MODEL_DATA_DIR cor.parmdb=$pathMS/cal-c'+str(c)+'.h5 cor.direction=['+d.name+']', \
# Smooth DATA -> SMOOTHED_DATA
logger.info('BL-based smoothing...')
MSs.run('BLsmooth.py -r -f 0.2 -i ' + incol + ' -o SMOOTHED_DATA $pathMS',
log='$nameMS_smooth1-c' + str(c) + '.log',
commandType='python')
# solve PH - group*_TC.MS:SMOOTHED_DATA
logger.info('Solving PH...')
MSs.run('DPPP '+parset_dir+'/DPPP-solPH.parset msin=$pathMS sol.h5parm=$pathMS/ph.h5', \
log='$nameMS_solPH-c'+str(c)+'.log', commandType='DPPP')
# LoSoTo
for MS in MSs.getListObj():
lib_util.run_losoto(s, 'ph-c' + str(c) + '-' + MS.nameMS,
MS.pathMS + '/ph.h5',
[parset_dir + '/losoto-ph.parset'])
os.system('mv plots-ph-c' + str(c) + '* self/plots/')
s.add('H5parm_collector.py -V -s sol000 -o self/solutions/cal-ph-c'+str(c)+'.h5 '+' '.join(glob.glob('cal-ph-c'+str(c)+'*.h5')),\
log='losotoPH-c'+str(c)+'.log', commandType="python", processors='max')
s.run(check=True)
lib_util.check_rm('cal-ph-c' + str(c) + '*.h5')
# # correct DELAY - group*_TC.MS:(SUBTRACTED_)DATA -> group*_TC.MS:CORRECTED_DATA
# logger.info('Correcting Delay...')
# MSs.run('DPPP '+parset_dir+'/DPPP-cor.parset msin=$pathMS msin.datacolumn='+incol+' cor.parmdb=self/solutions/cal-ph-c'+str(c)+'.h5 cor.correction=clock000', \
# log='$nameMS_corDELAY-c'+str(c)+'.log', commandType='DPPP')
# #MSs.run('DPPP '+parset_dir+'/DPPP-cor.parset msin=$pathMS msin.datacolumn=CORRECTED_DATA cor.parmdb=self/solutions/cal-ph-c'+str(c)+'.h5 cor.correction=phase000', \
# # log='$nameMS_corDELAY-c'+str(c)+'.log', commandType='DPPP')
#
# # Smooth CORRECTED_DATA -> SMOOTHED_DATA
(skymodel_voro, skymodel_voro_skydb),
log='makesourcedb_voro.log',
commandType='general')
s.run(check=True)
del lsm
################################################################
# Calibration
logger.info('Calibrating...')
MSs.run('DPPP '+parset_dir+'/DPPP-solDD.parset msin=$pathMS ddecal.h5parm=$pathMS/cal-c'+str(c)+'.h5 ddecal.sourcedb='+skymodel_cl_skydb, \
log='$nameMS_solDD-c'+str(c)+'.log', commandType='DPPP')
# Plot solutions
lib_util.run_losoto(
s, 'c' + str(c),
[MS + '/cal-c' + str(c) + '.h5'
for MS in MSs.getListStr()], [parset_dir + '/losoto-plot.parset'])
os.system('mv plots-c' + str(c) + '* ddcal/plots')
##############################################################
# low S/N DIE corrections
# TODO: add amp and FR sol + correction here after ft() a DDE-corrupted model
###########################################################
# Empty the dataset
logger.info('Set SUBTRACTED_DATA = DATA...')
MSs.run('taql "update $pathMS set SUBTRACTED_DATA = DATA"',
log='$nameMS_taql1-c' + str(c) + '.log',
commandType='general')
logger.info('Subtraction...')
logger.info('Remove bad timestamps...')
MSs.run('flagonmindata.py -f 0.5 $pathMS',
log='$nameMS_flagonmindata.log',
commandType='python')
####################################################
# 1: Solving
if w.todo('calib-ph-c%02i' % c):
# solve G - group*_TC.MS:CORRECTED_DATA
logger.info('Solving fast...')
solint = next(solint_ph)
MSs.run('DPPP ' + parset_dir + '/DPPP-solG.parset msin=$pathMS msin.datacolumn=DATA sol.h5parm=$pathMS/calGp.h5 sol.mode=scalarcomplexgain \
sol.solint=' +str(solint)+' sol.smoothnessconstraint=5e6', \
log='$nameMS_solGp-c'+str(c)+'.log', commandType="DPPP")
lib_util.run_losoto(s, 'Gp-c'+str(c), [ms+'/calGp.h5' for ms in MSs.getListStr()], \
[parset_dir+'/losoto-plot2d.parset', parset_dir+'/losoto-plot.parset'])
# Correct DATA -> CORRECTED_DATA
logger.info('Correction PH...')
MSs.run('DPPP ' + parset_dir + '/DPPP-cor.parset msin=$pathMS msin.datacolumn=DATA cor.parmdb=cal-Gp-c'+str(c)+'.h5 cor.correction=phase000', \
log='$nameMS_corPH-c'+str(c)+'.log', commandType='DPPP')
w.done('calib-ph-c%02i' % c)
### DONE
if doamp:
if w.todo('calib-amp-c%02i' % c):
# solve G - group*_TC.MS:CORRECTED_DATA
logger.info('Solving slow...')
#sol.antennaconstraint=[[CSsuperLBA,CS001LBA,CS002LBA,CS003LBA,CS004LBA,CS005LBA,CS006LBA,CS007LBA,CS011LBA,CS013LBA,CS017LBA,CS021LBA,CS024LBA,CS026LBA,CS028LBA,CS030LBA,CS031LBA,CS032LBA,CS101LBA,CS103LBA,CS201LBA,CS301LBA,CS302LBA,CS401LBA,CS501LBA,RS106LBA,RS205LBA,RS208LBA,RS210LBA,RS305LBA,RS306LBA,RS307LBA,RS310LBA,RS406LBA,RS407LBA,RS409LBA,RS503LBA,RS508LBA,RS509LBA]] \
incol = 'DATA'
# Smooth DATA -> SMOOTHED_DATA
logger.info('BL-based smoothing...')
MSs.run('BLsmooth.py -r -f 0.2 -i ' + incol + ' -o SMOOTHED_DATA $pathMS',
log='$nameMS_smooth1-c' + str(c) + '.log',
commandType='python')
# solve TEC - group*_TC.MS:SMOOTHED_DATA
logger.info('Solving TEC...')
MSs.run('DPPP '+parset_dir+'/DPPP-solTECdd.parset msin=$pathMS ddecal.h5parm=$pathMS/tec.h5', \
log='$nameMS_solTEC-c'+str(c)+'.log', commandType='DPPP')
# LoSoTo plot
lib_util.run_losoto(s, 'tec' + str(c),
[MS + '/tec.h5' for MS in MSs.getListStr()],
[parset_dir + '/losoto-plot-tec.parset'])
os.system('mv plots-tec' + str(c) + '* self/solutions/')
os.system('mv cal-tec' + str(c) + '*.h5 self/solutions/')
# correct TEC - group*_TC.MS:(SUBTRACTED_)DATA -> group*_TC.MS:CORRECTED_DATA
logger.info('Correcting TEC...')
MSs.run('DPPP '+parset_dir+'/DPPP-corTEC.parset msin=$pathMS msin.datacolumn='+incol+' cor1.parmdb=$pathMS/tec.h5 cor2.parmdb=$pathMS/tec.h5', \
log='$nameMS_corTEC-c'+str(c)+'.log', commandType='DPPP')
#####################################################################################################
# Faraday rotation correction
#if c >= 1:
#
# # To circular - SB.MS:CORRECTED_DATA -> SB.MS:CORRECTED_DATA (circular)
# logger.info('Convert to circular...')
# Smooth data DATA -> SMOOTHED_DATA (BL-based smoothing)
# TODO: gmrt amplitude are unstable, the smooth seems not to work fine
#logger.info('BL-smooth...')
#MSs_cals.run('BLsmooth.py -r -i DATA -o SMOOTHED_DATA $pathMS', log='$nameMS_smooth1.log', commandType ='python', maxThreads=10)
# Solve cal_SB.MS:SMOOTHED_DATA (only solve)
logger.info('Calibrating...')
MSs_cals.run(
'DPPP ' + parset_dir +
'/DPPP-soldd.parset msin=$pathMS msin.datacolumn=DATA sol.h5parm=$pathMS/diag.h5 sol.mode=diagonal',
log='$nameMS_sol.log',
commandType="DPPP")
lib_util.run_losoto(s, 'diag', [ms+'/diag.h5' for ms in MSs_cals.getListStr()], \
[parset_dir+'/losoto-plot-ph.parset', parset_dir+'/losoto-plot-amp.parset', parset_dir+'/losoto-flag.parset', \
parset_dir+'/losoto-pa.parset', parset_dir+'/losoto-iono.parset', parset_dir+'/losoto-bp.parset'])
# Transfer to target
MSs_tgts = lib_ms.AllMSs(glob.glob('tgts/*/mss/%02i*MS' % msID), s)
logger.info('Add columns...')
MSs_tgts.run('addcol2ms.py -m $pathMS -c CORRECTED_DATA -i DATA',
log="$nameMS_addcol.log",
commandType="python")
# Trasnfer tgt_SB.MS:DATA -> tgt.MS:CORRECTED_DATA
logger.info('Correcting targets...')
MSs_tgts.run('DPPP ' + parset_dir + '/DPPP-cor.parset msin=$pathMS',
log='$nameMS_cor.log',
commandType="DPPP")
####################################################
# 1: Solving
# Smooth DATA -> SMOOTHED_DATA
logger.info('BL-based smoothing...')
MSs.run('BLsmooth.py -r -f 0.2 -i DATA -o SMOOTHED_DATA $pathMS',
log='$nameMS_smooth1.log',
commandType='python')
# First get scalar solution to get ph for CS
# solve G - group*_TC.MS:SMOOTHED_DATA
logger.info('Solving 1...')
MSs.run('DPPP ' + parset_dir + '/DPPP-solG.parset msin=$pathMS msin.datacolumn=SMOOTHED_DATA sol.h5parm=$pathMS/calG0.h5 sol.mode=diagonal \
sol.antennaconstraint=[[CS002LBA,CS003LBA,CS004LBA,CS005LBA,CS006LBA,CS007LBA]]' , \
log='$nameMS_solG0-c'+str(c)+'.log', commandType="DPPP")
lib_util.run_losoto(s, 'G0-c'+str(c), [ms+'/calG0.h5' for ms in MSs.getListStr()], \
[parset_dir+'/losoto-plot-ph.parset', parset_dir+'/losoto-plot-amp.parset', parset_dir+'/losoto-resetremote.parset'])
# Pre-apply ph on CS and fix all of them to get ph for RS
# solve G - group*_TC.MS:SMOOTHED_DATA
logger.info('Solving 2...')
MSs.run('DPPP ' + parset_dir + '/DPPP-solG.parset msin=$pathMS msin.datacolumn=SMOOTHED_DATA sol.h5parm=$pathMS/calG1.h5 sol.mode=diagonal \
sol.antennaconstraint=[[CS001LBA,CS002LBA,CS003LBA,CS004LBA,CS005LBA,CS006LBA,CS007LBA,CS011LBA,CS013LBA,CS017LBA,CS021LBA,CS024LBA,CS026LBA,CS028LBA,CS030LBA,CS031LBA,CS032LBA,CS101LBA,CS103LBA,CS201LBA,CS301LBA,CS302LBA,CS401LBA,CS501LBA]] \
sol.applycal.parmdb=cal-G0-c' +str(c)+'.h5 sol.applycal.correction=phase000', \
log='$nameMS_solG1-c'+str(c)+'.log', commandType="DPPP")
lib_util.run_losoto(s, 'G1-c'+str(c), [ms+'/calG1.h5' for ms in MSs.getListStr()], \
[parset_dir+'/losoto-plot-ph.parset', parset_dir+'/losoto-plot-amp.parset', parset_dir+'/losoto-fr.parset'])
# Convert to linear CORRECTED_DATA -> CORRECTED_DATA
logger.info('Converting to linear...')
MSs.run('mslin2circ.py -r -i $pathMS:DATA -o $pathMS:CORRECTED_DATA',
log='$nameMS_circ2lin.log',
log='$nameMS_pre.log',
commandType='DPPP')
#####################################################################################################
# Self-cal cycle
for c in range(3):
logger.info('Start selfcal cycle: ' + str(c))
# solve - concat*.MS:SMOOTHED_DATA
if not is_wideband:
logger.info('Solving G...')
MSs.run('DPPP '+parset_dir+'/DPPP-solG.parset msin=$pathMS msin.datacolumn=DATA sol.h5parm=$pathMS/gs.h5 \
sol.solint=1 sol.nchan=1 sol.mode=complexgain sol.smoothnessconstraint=1e6' , \
log='$nameMS_solG-c'+str(c)+'.log', commandType='DPPP')
lib_util.run_losoto(s, 'gs'+str(c), [MS+'/gs.h5' for MS in MSs.getListStr()], \
[parset_dir+'/losoto-plot-ph.parset', parset_dir+'/losoto-plot-amp.parset', parset_dir+'/losoto-flag.parset', parset_dir+'/losoto-norm.parset'])
os.system('mv plots-gs' + str(c) + '* self/solutions/')
os.system('mv cal-gs' + str(c) + '*.h5 self/solutions/')
# correct phases - MS:DATA -> MS:CORRECTED_DATA
MSs.run('DPPP '+parset_dir+'/DPPP-cor.parset msin=$pathMS cor.parmdb=self/solutions/cal-gs'+str(c)+'.h5 cor.correction=phase000', \
log='$nameMS_corGp-c'+str(c)+'.log', commandType='DPPP')
if c >= 2:
# correct amplitudes - MS:DATA -> MS:CORRECTED_DATA
logger.info('Correcting Ga...')
MSs.run('DPPP '+parset_dir+'/DPPP-cor.parset msin=$pathMS msin.datacolumn=CORRECTED_DATA cor.parmdb=self/solutions/cal-gs'+str(c)+'.h5 cor.correction=amplitude000', \
log='$nameMS_corGa-c'+str(c)+'.log', commandType='DPPP')
else:
logger.info('Solving TEC...')
MSs.run('DPPP '+parset_dir+'/DPPP-solG.parset msin=$pathMS msin.datacolumn=DATA sol.h5parm=$pathMS/tec.h5 \
sol.solint=1 sol.nchan=1 sol.mode=tec' , \
log='$nameMS_solG-c'+str(c)+'.log', commandType='DPPP')
logger.info('BL-based smoothing...')
MSs.run(
'BLsmooth.py -f 1.0 -r -i SUBTRACTED_DATA -o SMOOTHED_DATA $pathMS',
log='$nameMS_smooth-c' + str(c) + '.log',
commandType='python')
# Calibration - ms:SMOOTHED_DATA
logger.info('Core calibration...')
MSs.run('DPPP '+parset_dir+'/DPPP-solTEC.parset msin=$pathMS msin.baseline="[CR]*&&;!RS210LBA;!RS310LBA;!RS509LBA;!RS508LBA;!RS409LBA;!RS208LBA;!RS307LBA" \
sol.antennaconstraint=[[CS002LBA,CS003LBA,CS004LBA,CS005LBA,CS006LBA,CS007LBA]] \
sol.solint=20 sol.nchan=40 sol.h5parm=$pathMS/cal-core-c' +str(c)+'.h5 sol.sourcedb='+skymodel_cl_skydb, \
log='$nameMS_solTECcore-c'+str(c)+'.log', commandType='DPPP')
# Plot solutions
for MS in MSs.getListObj():
lib_util.run_losoto(s, 'core-c'+str(c)+'-'+MS.nameMS, MS.pathMS+'/cal-core-c'+str(c)+'.h5', \
[parset_dir+'/losoto-resetremote.parset', parset_dir+'/losoto-plot-tec.parset'])
os.system('mv plots-core-c' + str(c) + '* ddcal/plots')
s.add('H5parm_collector.py -V -s sol000 -o ddcal/solutions/cal-core-c'+str(c)+'.h5 '+' '.join(glob.glob('cal-core-c'+str(c)+'-*.h5')),\
log='losoto-collector-c'+str(c)+'.log', commandType="python", processors='max')
s.run(check=True)
lib_util.check_rm('cal-core-c' + str(c) + '-*.h5')
# Calibration - ms:SMOOTHED_DATA
logger.info('Remote calibration...')
MSs.run('DPPP '+parset_dir+'/DPPP-solTEC.parset msin=$pathMS \
sol.antennaconstraint=[[CS001LBA,CS002LBA,CS003LBA,CS004LBA,CS005LBA,CS006LBA,CS007LBA,CS011LBA,CS013LBA,CS017LBA,CS021LBA,CS024LBA,CS026LBA,CS028LBA,CS030LBA,CS031LBA,CS032LBA,CS101LBA,CS103LBA,CS201LBA,CS301LBA,CS302LBA,CS401LBA,CS501LBA,RS106LBA,RS205LBA,RS305LBA,RS306LBA,RS406LBA,RS407LBA,RS503LBA]] \
sol.applycal.parmdb=ddcal/solutions/cal-core-c' +str(c)+'.h5 sol.applycal.correction=tec000 \
sol.solint=10 sol.nchan=4 sol.h5parm=$pathMS/cal-remote-c' +str(c)+'.h5 sol.sourcedb='+skymodel_cl_skydb, \
log='$nameMS_solTECremote-c'+str(c)+'.log', commandType='DPPP')
# Plot solutions
for c in range(3):
logger.info(f'Cycle {c}: {threads[c]} thread(s) @ {cpu_per_thread[c]} cpu')
s = lib_util.Scheduler(log_dir=logger_obj.log_dir,
dry=False,
maxThreads=threads[c])
# solve TEC - ms:SMOOTHED_DATA
logger.info('Solving TEC...')
start = time.time()
MSs.run('DPPP ' + parset_dir +
'/DPPP-solTEC.parset msin=$pathMS sol.h5parm=$pathMS/tec.h5 \
sol.antennaconstraint=[[CS001LBA,CS002LBA,CS003LBA,CS004LBA,CS005LBA,CS006LBA,CS007LBA]]',
log=f'$nameMS_solTEC-c{c}.log',
commandType='DPPP') #maxThreads
logger.info(f'DP3: {start-time.time()}')
lib_util.run_losoto(s, 'tec-c' + str(c),
[ms + '/tec.h5' for ms in MSs.getListStr()],
[parset_dir + '/losoto-plot-tec.parset'])
logger.info(f'DP3 + LoSoTo: {start-time.time()}')
os.system('mv cal-tec-c' + str(c) + '.h5 bench/')
os.system('mv plots-tec-c' + str(c) + ' bench/plots/')
sys.exit()
#####################################################################################################
# Self-cal cycle
for c in range(2):
logger.info('Start selfcal cycle: ' + str(c))
if c == 0:
with w.if_todo('set_corrected_data'):
logger.info('Set CORRECTED_DATA = DATA...')
MSs.run('taql "update $pathMS set CORRECTED_DATA = DATA"',
# Smoothing - ms:SUBTRACTED_DATA -> ms:SMOOTHED_DATA
logger.info('BL-based smoothing...')
MSs.run(
'BLsmooth.py -f 1.0 -r -i SUBTRACTED_DATA -o SMOOTHED_DATA $pathMS',
log='$nameMS_smooth-c' + str(c) + '.log',
commandType='python')
# Calibration - ms:SMOOTHED_DATA
logger.info('Calibrating...')
MSs.run('DPPP '+parset_dir+'/DPPP-solDD.parset msin=$pathMS ddecal.h5parm=$pathMS/cal-c'+str(c)+'.h5 ddecal.sourcedb='+skymodel_cl_skydb, \
log='$nameMS_solDD-c'+str(c)+'.log', commandType='DPPP')
# Plot solutions
lib_util.run_losoto(
s, 'c' + str(c),
[MS + '/cal-c' + str(c) + '.h5'
for MS in MSs.getListStr()], [parset_dir + '/losoto-plot.parset'])
os.system('mv plots-c' + str(c) + '* ddcal/plots')
##############################################################
# low S/N DIE corrections
# TODO: add a higher low-uv cut to remove galaxy effect
if c >= 0:
logger.info('DIE calibration...')
# predict and corrupt each facet
logger.info('Reset MODEL_DATA...')
MSs.run('taql "update $pathMS set MODEL_DATA = 0"',
log='$nameMS_taql-c' + str(c) + '.log',
commandType='general')
for i, d in enumerate(directions):
