# Smoothing - ms:SUBTRACTED_DATA -> ms:SMOOTHED_DATA logger.info('BL-based smoothing...') MSs.run( 'BLsmooth.py -c 8 -r -i SUBTRACTED_DATA -o SMOOTHED_DATA $pathMS', log='$nameMS_smooth-c' + str(c) + '.log', commandType='python') # Calibration - ms:SMOOTHED_DATA logger.info('Gain calibration...') MSs.run('DPPP '+parset_dir+'/DPPP-solG.parset msin=$pathMS \ sol.h5parm=$pathMS/cal-g-c' +str(c)+'.h5 sol.sourcedb='+skymodel_cl_skydb, \ log='$nameMS_solG-c'+str(c)+'.log', commandType='DPPP') # Plot solutions lib_util.run_losoto(s, 'g-c'+str(c), [ms+'/cal-g-c'+str(c)+'.h5' for ms in MSs.getListStr()], \ [parset_dir+'/losoto-amp.parset', parset_dir+'/losoto-plot-amp.parset', parset_dir+'/losoto-plot-ph.parset']) os.system('mv plots-g-c' + str(c) + ' ddcal/plots') os.system('mv cal-g-c' + str(c) + '.h5 ddcal/solutions') ### DONE ########################################################### # use idg and A-term to correct the data, single image if aterm_imaging: #wsclean -mem 90.0 -scale 0.0004166666666666667 -aterm-config /beegfs/rafferty/Data/LOFAR/Screens/Factor_sim/pipelines/image_1/sector_3/chunk9.ms.make_aterm_config -multiscale-scales 0 -size 1500 1500 -deconvolution-channels 4 -fits-mask /beegfs/rafferty/Data/LOFAR/Screens/Factor_sim/pipelines/image_1/sector_3/chunk9.ms.premask -j 6 -auto-mask 3.6 -idg-mode hybrid -channels-out 12 -local-rms-window 50 -mgain 0.5 -minuv-l 80.0 -fit-spectral-pol 3 -maxuv-l 1000000.0 -weighting-rank-filter 3 -aterm-kernel-size 32 -temp-dir /tmp -name /beegfs/rafferty/Data/LOFAR/Screens/Factor_sim/pipelines/image_1/sector_3/chunk9.ms.image -padding 1.2 -pol I -multiscale-shape gaussian -auto-threshold 1.0 -local-rms-method rms-with-min -weight briggs -0.5 -niter 13635 -no-update-model-required -multiscale -fit-beam -reorder -save-source-list -local-rms -join-channels -use-idg -apply-primary-beam -nmiter 4 lib_util.run_wsclean(s, 'wscleanDD-c%02i.log' %c, MSs.getStrWsclean(), name='img/wideDD-c%02i' %c, save_source_list='', size=imsize, scale=str(pixscale)+'arcsec', \ weight=weight, niter=100000, no_update_model_required='', minuv_l=30, maxuv_l=maxuv_l, mgain=0.85, \ use_idg='', grid_with_beam='', use_differential_lofar_beam='', beam_aterm_update=400, \ multiscale='', multiscale_scales='0,10,20,40,80', \ parallel_deconvolution=512, local_rms='', auto_threshold=0.75, auto_mask=1.5, fits_mask=im.maskname, \
# Smooth data DATA -> SMOOTHED_DATA (BL-based smoothing) logger.info('BL-smooth...') MSs.run('BLsmooth.py -r -c 1 -n 8 -i DATA -o SMOOTHED_DATA $pathMS', log='$nameMS_smooth1.log', commandType='python', maxThreads=8) # Solve cal_SB.MS:SMOOTHED_DATA (only solve) logger.info('Calibrating PA...') MSs.run( 'DPPP ' + parset_dir + '/DPPP-soldd.parset msin=$pathMS sol.h5parm=$pathMS/pa.h5 sol.mode=rotation+diagonal', log='$nameMS_solPA.log', commandType="DPPP") lib_util.run_losoto(s, 'pa', [ms+'/pa.h5' for ms in MSs.getListStr()], \ [parset_dir+'/losoto-plot-ph.parset', parset_dir+'/losoto-plot-rot.parset', parset_dir+'/losoto-plot-amp.parset', parset_dir+'/losoto-pa.parset']) # Pol align correction DATA -> CORRECTED_DATA logger.info('Polalign correction...') MSs.run( 'DPPP ' + parset_dir + '/DPPP-cor.parset msin=$pathMS msin.datacolumn=DATA cor.parmdb=cal-pa.h5 cor.correction=polalign', log='$nameMS_corPA.log', commandType="DPPP") w.done('cal_pa') ### DONE ######################################################## # 2: find FR
#################################################### # 1: Solving with w.if_todo('calib-ph-c%02i' % c): # solve G - group*_TC.MS:CORRECTED_DATA logger.info('Solving fast...') solint = next(solint_ph) MSs.run( 'DPPP ' + parset_dir + '/DPPP-solG.parset msin=$pathMS msin.datacolumn=DATA sol.h5parm=$pathMS/calGp.h5 sol.mode=scalarcomplexgain \ sol.solint=' + str(solint) + ' sol.smoothnessconstraint=5e6', log='$nameMS_solGp-c' + str(c) + '.log', commandType="DPPP") lib_util.run_losoto(s, 'Gp-c' + str(c), [ms + '/calGp.h5' for ms in MSs.getListStr()], [ parset_dir + '/losoto-clip-large.parset', parset_dir + '/losoto-plot2d.parset', parset_dir + '/losoto-plot.parset' ]) # Correct DATA -> CORRECTED_DATA logger.info('Correction PH...') MSs.run('DPPP ' + parset_dir + '/DPPP-cor.parset msin=$pathMS msin.datacolumn=DATA cor.parmdb=cal-Gp-c'+str(c)+'.h5 cor.correction=phase000', \ log='$nameMS_corPH-c'+str(c)+'.log', commandType='DPPP') ### DONE if doamp: with w.if_todo('calib-amp-c%02i' % c): # solve G - group*_TC.MS:CORRECTED_DATA logger.info('Solving slow...') #sol.antennaconstraint=[[CSsuperLBA,CS001LBA,CS002LBA,CS003LBA,CS004LBA,CS005LBA,CS006LBA,CS007LBA,CS011LBA,CS013LBA,CS017LBA,CS021LBA,CS024LBA,CS026LBA,CS028LBA,CS030LBA,CS031LBA,CS032LBA,CS101LBA,CS103LBA,CS201LBA,CS301LBA,CS302LBA,CS401LBA,CS501LBA,RS106LBA,RS205LBA,RS208LBA,RS210LBA,RS305LBA,RS306LBA,RS307LBA,RS310LBA,RS406LBA,RS407LBA,RS409LBA,RS503LBA,RS508LBA,RS509LBA]] \
commandType='python') #################################################### # 1: find PA and remove it # Solve cal_SB.MS:DATA (only solve) if not os.path.exists('cal-pa-c0.h5'): logger.info('Solving PA...') MSs.run( 'DPPP ' + parset_dir + '/DPPP-soldd.parset msin=$pathMS msin.datacolumn=DATA sol.h5parm=$pathMS/pa.h5 sol.mode=rotation+diagonal \ sol.uvlambdarange=' + str(nouseblrange), log='$nameMS_solPA.log', commandType="DPPP") lib_util.run_losoto(s, 'pa-c'+str(c), [ms+'/pa.h5' for ms in MSs.getListStr()], \ [parset_dir+'/losoto-plot-ph.parset', parset_dir+'/losoto-plot-rot.parset', parset_dir+'/losoto-plot-amp.parset', parset_dir+'/losoto-pa.parset']) ################################################# # 2: find the FR and remve it # Beam correction DATA -> CORRECTED_DATA logger.info('PA correction...') MSs.run('DPPP '+parset_dir+'/DPPP-cor.parset msin=$pathMS msin.datacolumn=DATA cor.parmdb=cal-pa-c0.h5 cor.correction=polalign', \ log='$nameMS_corPA2.log', commandType="DPPP") # Beam correction CORRECTED_DATA -> CORRECTED_DATA logger.info('Beam correction...') MSs.run('DPPP ' + parset_dir + '/DPPP-beam.parset msin=$pathMS', log='$nameMS_beam2.log', commandType='DPPP')
solint_amp2 = next(iter_amp2_solint) if phSolMode == 'phase': logger.info('Phase calibration...') with w.if_todo('cal-ph-c%02i' % c): MSs.run( 'DPPP ' + parset_dir + '/DPPP-solG.parset msin=$pathMS msin.datacolumn=SMOOTHED_DATA sol.h5parm=$pathMS/cal-ph.h5 \ sol.mode=scalarphase sol.solint=' + str(solint_ph) + ' sol.smoothnessconstraint=5e6 \ sol.antennaconstraint=[[CS001LBA,CS002LBA,CS003LBA,CS004LBA,CS005LBA,CS006LBA,CS007LBA,CS011LBA,CS013LBA,CS017LBA,CS021LBA,CS024LBA,CS026LBA,CS028LBA,CS030LBA,CS031LBA,CS032LBA,CS101LBA,CS103LBA,CS201LBA,CS301LBA,CS302LBA,CS401LBA,CS501LBA]]', log='$nameMS_solGph-c%02i.log' % c, commandType='DPPP') lib_util.run_losoto(s, 'ph', [ms + '/cal-ph.h5' for ms in MSs.getListStr()], [parset_dir + '/losoto-plot1.parset'], plots_dir='extract/plots-%s' % c) os.system('mv cal-ph.h5 %s' % h5ph) with w.if_todo('cor-ph-c%02i' % c): # correct ph - ms:DATA -> ms:CORRECTED_DATA logger.info('Correct ph...') MSs.run( 'DPPP ' + parset_dir + '/DPPP-correct.parset msin=$pathMS msin.datacolumn=DATA msout.datacolumn=CORRECTED_DATA \ cor.parmdb=' + h5ph + ' cor.correction=phase000', log='$nameMS_correct-c%02i.log' % c, commandType='DPPP') else: logger.info('Tecandphase calibration...')
dicomodel='%s.DicoModel' % rootimg_old, cache_dir='ddfcal/cache', sols_dir='ddfcal/solutions', clusterfile=clusterfile, CovQ=0.02, niterkf=6, uvrange=uvrange, wtuv=None, robust=image_robust, dt=dt, NChanSols=12, MergeSmooth=True) # run losoto lib_util.run_losoto(s, 'g-c'+str(c), glob.glob('ddfcal/solutions/TC*.MS/*npz'), \ [parset_dir+'/losoto-plot-amp.parset', parset_dir+'/losoto-plot-ph.parset']) os.system('mv plots-g-c' + str(c) + ' ddfcal/plots') os.system('mv cal-g-c' + str(c) + '.h5 ddfcal/solutions') # image if c <= 1: applysols = 'P' else: applysols = 'P' logger.info('Imaging...') lib_ddfacet.ddf_image(s, 'ddf%02i.log' % c, MSs, rootimg,cleanmode='SSD',\ ddsols=solname,applysols=applysols,\ majorcycles=2,robust=image_robust,\ peakfactor=0.001,colname='DATA',\ automask=True,automask_threshold=10, cleanmask=maskname, \ normalization='BLBased',apply_weights=True,use_weightspectrum=False,uvrange=uvrange, use_dicomodel=True,dicomodel_base=rootimg_old,
# Calibration - ms:SMOOTHED_DATA # possible to put nchan=6 if less channels are needed in the h5parm (e.g. for IDG) logger.info('Gain phase calibration (solint: %i)...' % solint_ph) MSs_dir.run( 'DPPP ' + parset_dir + '/DPPP-solG.parset msin=$pathMS msin.datacolumn=SMOOTHED_DATA sol.h5parm=$pathMS/cal-ph.h5 \ sol.mode=scalarphase sol.solint=' + str(solint_ph) + ' sol.smoothnessconstraint=5e6 \ sol.antennaconstraint=[[CS001LBA,CS002LBA,CS003LBA,CS004LBA,CS005LBA,CS006LBA,CS007LBA,CS011LBA,CS013LBA,CS017LBA,CS021LBA,CS024LBA,CS026LBA,CS028LBA,CS030LBA,CS031LBA,CS032LBA,CS101LBA,CS103LBA,CS201LBA,CS301LBA,CS302LBA,CS401LBA,CS501LBA]]', log='$nameMS_solGph-' + logstringcal + '.log', commandType='DPPP') lib_util.run_losoto( s, 'ph', [ms + '/cal-ph.h5' for ms in MSs_dir.getListStr()], [parset_dir + '/losoto-plot1.parset'], plots_dir='ddcal/c%02i/plots/plots-%s' % (cmaj, logstringcal)) os.system('mv cal-ph.h5 %s' % d.get_h5parm('ph')) # correct ph - ms:DATA -> ms:CORRECTED_DATA logger.info('Correct ph...') MSs_dir.run( 'DPPP ' + parset_dir + '/DPPP-correct.parset msin=$pathMS msin.datacolumn=DATA msout.datacolumn=CORRECTED_DATA \ cor.parmdb=' + d.get_h5parm('ph') + ' cor.correction=phase000', log='$nameMS_correct-' + logstringcal + '.log', commandType='DPPP') if doamp:
#parset = lib_util.getParset() parset_dir = '/home/baq1889/scripts/LiLF/parsets/LOFAR_dd' ############################################################# MSs = lib_ms.AllMSs( glob.glob('*MS'), s, check_flags=False ) skymodel_voro_skydb = 'skymodel00_voro.skydb' c=0 # Calibration - ms:SMOOTHED_DATA logger.info('Calibrating...') MSs.run('DPPP '+parset_dir+'/DPPP-solDD.parset msin=$pathMS ddecal.h5parm=$pathMS/cal-core.h5 \ ddecal.sourcedb='+skymodel_cl_skydb+' ddecal.solint=15 ddecal.nchan=30', \ log='$nameMS_solDD-core.log', commandType='DPPP') lib_util.run_losoto(s, 'core', [MS+'/cal-core.h5' for MS in MSs.getListStr()], \ [parset_dir+'/losoto-core.parset']) logger.info('re-calibration...') # predict and corrupt each facet logger.info('Reset MODEL_DATA...') MSs.run('taql "update $pathMS set MODEL_DATA = 0"', log='$nameMS_taql-c'+str(c)+'.log', commandType='general') for i, d in enumerate(directions): # predict - ms:MODEL_DATA logger.info('Patch '+d.name+': predict...') MSs.run('DPPP '+parset_dir+'/DPPP-predict.parset msin=$pathMS msout.datacolumn=MODEL_DATA_DIR pre.sourcedb='+skymodel_voro_skydb+' pre.sources='+d.name, \ log='$nameMS_pre1-c'+str(c)+'-'+d.name+'.log', commandType='DPPP') # corrupt - ms:MODEL_DATA -> ms:MODEL_DATA logger.info('Patch '+d.name+': corrupt...') MSs.run('DPPP '+parset_dir+'/DPPP-corrupt.parset msin=$pathMS msin.datacolumn=MODEL_DATA_DIR msout.datacolumn=MODEL_DATA_DIR cor.parmdb=$pathMS/cal-c'+str(c)+'.h5 cor.direction=['+d.name+']', \
# Smooth DATA -> SMOOTHED_DATA logger.info('BL-based smoothing...') MSs.run('BLsmooth.py -r -f 0.2 -i ' + incol + ' -o SMOOTHED_DATA $pathMS', log='$nameMS_smooth1-c' + str(c) + '.log', commandType='python') # solve PH - group*_TC.MS:SMOOTHED_DATA logger.info('Solving PH...') MSs.run('DPPP '+parset_dir+'/DPPP-solPH.parset msin=$pathMS sol.h5parm=$pathMS/ph.h5', \ log='$nameMS_solPH-c'+str(c)+'.log', commandType='DPPP') # LoSoTo for MS in MSs.getListObj(): lib_util.run_losoto(s, 'ph-c' + str(c) + '-' + MS.nameMS, MS.pathMS + '/ph.h5', [parset_dir + '/losoto-ph.parset']) os.system('mv plots-ph-c' + str(c) + '* self/plots/') s.add('H5parm_collector.py -V -s sol000 -o self/solutions/cal-ph-c'+str(c)+'.h5 '+' '.join(glob.glob('cal-ph-c'+str(c)+'*.h5')),\ log='losotoPH-c'+str(c)+'.log', commandType="python", processors='max') s.run(check=True) lib_util.check_rm('cal-ph-c' + str(c) + '*.h5') # # correct DELAY - group*_TC.MS:(SUBTRACTED_)DATA -> group*_TC.MS:CORRECTED_DATA # logger.info('Correcting Delay...') # MSs.run('DPPP '+parset_dir+'/DPPP-cor.parset msin=$pathMS msin.datacolumn='+incol+' cor.parmdb=self/solutions/cal-ph-c'+str(c)+'.h5 cor.correction=clock000', \ # log='$nameMS_corDELAY-c'+str(c)+'.log', commandType='DPPP') # #MSs.run('DPPP '+parset_dir+'/DPPP-cor.parset msin=$pathMS msin.datacolumn=CORRECTED_DATA cor.parmdb=self/solutions/cal-ph-c'+str(c)+'.h5 cor.correction=phase000', \ # # log='$nameMS_corDELAY-c'+str(c)+'.log', commandType='DPPP') # # # Smooth CORRECTED_DATA -> SMOOTHED_DATA
(skymodel_voro, skymodel_voro_skydb), log='makesourcedb_voro.log', commandType='general') s.run(check=True) del lsm ################################################################ # Calibration logger.info('Calibrating...') MSs.run('DPPP '+parset_dir+'/DPPP-solDD.parset msin=$pathMS ddecal.h5parm=$pathMS/cal-c'+str(c)+'.h5 ddecal.sourcedb='+skymodel_cl_skydb, \ log='$nameMS_solDD-c'+str(c)+'.log', commandType='DPPP') # Plot solutions lib_util.run_losoto( s, 'c' + str(c), [MS + '/cal-c' + str(c) + '.h5' for MS in MSs.getListStr()], [parset_dir + '/losoto-plot.parset']) os.system('mv plots-c' + str(c) + '* ddcal/plots') ############################################################## # low S/N DIE corrections # TODO: add amp and FR sol + correction here after ft() a DDE-corrupted model ########################################################### # Empty the dataset logger.info('Set SUBTRACTED_DATA = DATA...') MSs.run('taql "update $pathMS set SUBTRACTED_DATA = DATA"', log='$nameMS_taql1-c' + str(c) + '.log', commandType='general') logger.info('Subtraction...')
logger.info('Remove bad timestamps...') MSs.run('flagonmindata.py -f 0.5 $pathMS', log='$nameMS_flagonmindata.log', commandType='python') #################################################### # 1: Solving if w.todo('calib-ph-c%02i' % c): # solve G - group*_TC.MS:CORRECTED_DATA logger.info('Solving fast...') solint = next(solint_ph) MSs.run('DPPP ' + parset_dir + '/DPPP-solG.parset msin=$pathMS msin.datacolumn=DATA sol.h5parm=$pathMS/calGp.h5 sol.mode=scalarcomplexgain \ sol.solint=' +str(solint)+' sol.smoothnessconstraint=5e6', \ log='$nameMS_solGp-c'+str(c)+'.log', commandType="DPPP") lib_util.run_losoto(s, 'Gp-c'+str(c), [ms+'/calGp.h5' for ms in MSs.getListStr()], \ [parset_dir+'/losoto-plot2d.parset', parset_dir+'/losoto-plot.parset']) # Correct DATA -> CORRECTED_DATA logger.info('Correction PH...') MSs.run('DPPP ' + parset_dir + '/DPPP-cor.parset msin=$pathMS msin.datacolumn=DATA cor.parmdb=cal-Gp-c'+str(c)+'.h5 cor.correction=phase000', \ log='$nameMS_corPH-c'+str(c)+'.log', commandType='DPPP') w.done('calib-ph-c%02i' % c) ### DONE if doamp: if w.todo('calib-amp-c%02i' % c): # solve G - group*_TC.MS:CORRECTED_DATA logger.info('Solving slow...') #sol.antennaconstraint=[[CSsuperLBA,CS001LBA,CS002LBA,CS003LBA,CS004LBA,CS005LBA,CS006LBA,CS007LBA,CS011LBA,CS013LBA,CS017LBA,CS021LBA,CS024LBA,CS026LBA,CS028LBA,CS030LBA,CS031LBA,CS032LBA,CS101LBA,CS103LBA,CS201LBA,CS301LBA,CS302LBA,CS401LBA,CS501LBA,RS106LBA,RS205LBA,RS208LBA,RS210LBA,RS305LBA,RS306LBA,RS307LBA,RS310LBA,RS406LBA,RS407LBA,RS409LBA,RS503LBA,RS508LBA,RS509LBA]] \
incol = 'DATA' # Smooth DATA -> SMOOTHED_DATA logger.info('BL-based smoothing...') MSs.run('BLsmooth.py -r -f 0.2 -i ' + incol + ' -o SMOOTHED_DATA $pathMS', log='$nameMS_smooth1-c' + str(c) + '.log', commandType='python') # solve TEC - group*_TC.MS:SMOOTHED_DATA logger.info('Solving TEC...') MSs.run('DPPP '+parset_dir+'/DPPP-solTECdd.parset msin=$pathMS ddecal.h5parm=$pathMS/tec.h5', \ log='$nameMS_solTEC-c'+str(c)+'.log', commandType='DPPP') # LoSoTo plot lib_util.run_losoto(s, 'tec' + str(c), [MS + '/tec.h5' for MS in MSs.getListStr()], [parset_dir + '/losoto-plot-tec.parset']) os.system('mv plots-tec' + str(c) + '* self/solutions/') os.system('mv cal-tec' + str(c) + '*.h5 self/solutions/') # correct TEC - group*_TC.MS:(SUBTRACTED_)DATA -> group*_TC.MS:CORRECTED_DATA logger.info('Correcting TEC...') MSs.run('DPPP '+parset_dir+'/DPPP-corTEC.parset msin=$pathMS msin.datacolumn='+incol+' cor1.parmdb=$pathMS/tec.h5 cor2.parmdb=$pathMS/tec.h5', \ log='$nameMS_corTEC-c'+str(c)+'.log', commandType='DPPP') ##################################################################################################### # Faraday rotation correction #if c >= 1: # # # To circular - SB.MS:CORRECTED_DATA -> SB.MS:CORRECTED_DATA (circular) # logger.info('Convert to circular...')
# Smooth data DATA -> SMOOTHED_DATA (BL-based smoothing) # TODO: gmrt amplitude are unstable, the smooth seems not to work fine #logger.info('BL-smooth...') #MSs_cals.run('BLsmooth.py -r -i DATA -o SMOOTHED_DATA $pathMS', log='$nameMS_smooth1.log', commandType ='python', maxThreads=10) # Solve cal_SB.MS:SMOOTHED_DATA (only solve) logger.info('Calibrating...') MSs_cals.run( 'DPPP ' + parset_dir + '/DPPP-soldd.parset msin=$pathMS msin.datacolumn=DATA sol.h5parm=$pathMS/diag.h5 sol.mode=diagonal', log='$nameMS_sol.log', commandType="DPPP") lib_util.run_losoto(s, 'diag', [ms+'/diag.h5' for ms in MSs_cals.getListStr()], \ [parset_dir+'/losoto-plot-ph.parset', parset_dir+'/losoto-plot-amp.parset', parset_dir+'/losoto-flag.parset', \ parset_dir+'/losoto-pa.parset', parset_dir+'/losoto-iono.parset', parset_dir+'/losoto-bp.parset']) # Transfer to target MSs_tgts = lib_ms.AllMSs(glob.glob('tgts/*/mss/%02i*MS' % msID), s) logger.info('Add columns...') MSs_tgts.run('addcol2ms.py -m $pathMS -c CORRECTED_DATA -i DATA', log="$nameMS_addcol.log", commandType="python") # Trasnfer tgt_SB.MS:DATA -> tgt.MS:CORRECTED_DATA logger.info('Correcting targets...') MSs_tgts.run('DPPP ' + parset_dir + '/DPPP-cor.parset msin=$pathMS', log='$nameMS_cor.log', commandType="DPPP")
#################################################### # 1: Solving # Smooth DATA -> SMOOTHED_DATA logger.info('BL-based smoothing...') MSs.run('BLsmooth.py -r -f 0.2 -i DATA -o SMOOTHED_DATA $pathMS', log='$nameMS_smooth1.log', commandType='python') # First get scalar solution to get ph for CS # solve G - group*_TC.MS:SMOOTHED_DATA logger.info('Solving 1...') MSs.run('DPPP ' + parset_dir + '/DPPP-solG.parset msin=$pathMS msin.datacolumn=SMOOTHED_DATA sol.h5parm=$pathMS/calG0.h5 sol.mode=diagonal \ sol.antennaconstraint=[[CS002LBA,CS003LBA,CS004LBA,CS005LBA,CS006LBA,CS007LBA]]' , \ log='$nameMS_solG0-c'+str(c)+'.log', commandType="DPPP") lib_util.run_losoto(s, 'G0-c'+str(c), [ms+'/calG0.h5' for ms in MSs.getListStr()], \ [parset_dir+'/losoto-plot-ph.parset', parset_dir+'/losoto-plot-amp.parset', parset_dir+'/losoto-resetremote.parset']) # Pre-apply ph on CS and fix all of them to get ph for RS # solve G - group*_TC.MS:SMOOTHED_DATA logger.info('Solving 2...') MSs.run('DPPP ' + parset_dir + '/DPPP-solG.parset msin=$pathMS msin.datacolumn=SMOOTHED_DATA sol.h5parm=$pathMS/calG1.h5 sol.mode=diagonal \ sol.antennaconstraint=[[CS001LBA,CS002LBA,CS003LBA,CS004LBA,CS005LBA,CS006LBA,CS007LBA,CS011LBA,CS013LBA,CS017LBA,CS021LBA,CS024LBA,CS026LBA,CS028LBA,CS030LBA,CS031LBA,CS032LBA,CS101LBA,CS103LBA,CS201LBA,CS301LBA,CS302LBA,CS401LBA,CS501LBA]] \ sol.applycal.parmdb=cal-G0-c' +str(c)+'.h5 sol.applycal.correction=phase000', \ log='$nameMS_solG1-c'+str(c)+'.log', commandType="DPPP") lib_util.run_losoto(s, 'G1-c'+str(c), [ms+'/calG1.h5' for ms in MSs.getListStr()], \ [parset_dir+'/losoto-plot-ph.parset', parset_dir+'/losoto-plot-amp.parset', parset_dir+'/losoto-fr.parset']) # Convert to linear CORRECTED_DATA -> CORRECTED_DATA logger.info('Converting to linear...') MSs.run('mslin2circ.py -r -i $pathMS:DATA -o $pathMS:CORRECTED_DATA', log='$nameMS_circ2lin.log',
log='$nameMS_pre.log', commandType='DPPP') ##################################################################################################### # Self-cal cycle for c in range(3): logger.info('Start selfcal cycle: ' + str(c)) # solve - concat*.MS:SMOOTHED_DATA if not is_wideband: logger.info('Solving G...') MSs.run('DPPP '+parset_dir+'/DPPP-solG.parset msin=$pathMS msin.datacolumn=DATA sol.h5parm=$pathMS/gs.h5 \ sol.solint=1 sol.nchan=1 sol.mode=complexgain sol.smoothnessconstraint=1e6' , \ log='$nameMS_solG-c'+str(c)+'.log', commandType='DPPP') lib_util.run_losoto(s, 'gs'+str(c), [MS+'/gs.h5' for MS in MSs.getListStr()], \ [parset_dir+'/losoto-plot-ph.parset', parset_dir+'/losoto-plot-amp.parset', parset_dir+'/losoto-flag.parset', parset_dir+'/losoto-norm.parset']) os.system('mv plots-gs' + str(c) + '* self/solutions/') os.system('mv cal-gs' + str(c) + '*.h5 self/solutions/') # correct phases - MS:DATA -> MS:CORRECTED_DATA MSs.run('DPPP '+parset_dir+'/DPPP-cor.parset msin=$pathMS cor.parmdb=self/solutions/cal-gs'+str(c)+'.h5 cor.correction=phase000', \ log='$nameMS_corGp-c'+str(c)+'.log', commandType='DPPP') if c >= 2: # correct amplitudes - MS:DATA -> MS:CORRECTED_DATA logger.info('Correcting Ga...') MSs.run('DPPP '+parset_dir+'/DPPP-cor.parset msin=$pathMS msin.datacolumn=CORRECTED_DATA cor.parmdb=self/solutions/cal-gs'+str(c)+'.h5 cor.correction=amplitude000', \ log='$nameMS_corGa-c'+str(c)+'.log', commandType='DPPP') else: logger.info('Solving TEC...') MSs.run('DPPP '+parset_dir+'/DPPP-solG.parset msin=$pathMS msin.datacolumn=DATA sol.h5parm=$pathMS/tec.h5 \ sol.solint=1 sol.nchan=1 sol.mode=tec' , \ log='$nameMS_solG-c'+str(c)+'.log', commandType='DPPP')
logger.info('BL-based smoothing...') MSs.run( 'BLsmooth.py -f 1.0 -r -i SUBTRACTED_DATA -o SMOOTHED_DATA $pathMS', log='$nameMS_smooth-c' + str(c) + '.log', commandType='python') # Calibration - ms:SMOOTHED_DATA logger.info('Core calibration...') MSs.run('DPPP '+parset_dir+'/DPPP-solTEC.parset msin=$pathMS msin.baseline="[CR]*&&;!RS210LBA;!RS310LBA;!RS509LBA;!RS508LBA;!RS409LBA;!RS208LBA;!RS307LBA" \ sol.antennaconstraint=[[CS002LBA,CS003LBA,CS004LBA,CS005LBA,CS006LBA,CS007LBA]] \ sol.solint=20 sol.nchan=40 sol.h5parm=$pathMS/cal-core-c' +str(c)+'.h5 sol.sourcedb='+skymodel_cl_skydb, \ log='$nameMS_solTECcore-c'+str(c)+'.log', commandType='DPPP') # Plot solutions for MS in MSs.getListObj(): lib_util.run_losoto(s, 'core-c'+str(c)+'-'+MS.nameMS, MS.pathMS+'/cal-core-c'+str(c)+'.h5', \ [parset_dir+'/losoto-resetremote.parset', parset_dir+'/losoto-plot-tec.parset']) os.system('mv plots-core-c' + str(c) + '* ddcal/plots') s.add('H5parm_collector.py -V -s sol000 -o ddcal/solutions/cal-core-c'+str(c)+'.h5 '+' '.join(glob.glob('cal-core-c'+str(c)+'-*.h5')),\ log='losoto-collector-c'+str(c)+'.log', commandType="python", processors='max') s.run(check=True) lib_util.check_rm('cal-core-c' + str(c) + '-*.h5') # Calibration - ms:SMOOTHED_DATA logger.info('Remote calibration...') MSs.run('DPPP '+parset_dir+'/DPPP-solTEC.parset msin=$pathMS \ sol.antennaconstraint=[[CS001LBA,CS002LBA,CS003LBA,CS004LBA,CS005LBA,CS006LBA,CS007LBA,CS011LBA,CS013LBA,CS017LBA,CS021LBA,CS024LBA,CS026LBA,CS028LBA,CS030LBA,CS031LBA,CS032LBA,CS101LBA,CS103LBA,CS201LBA,CS301LBA,CS302LBA,CS401LBA,CS501LBA,RS106LBA,RS205LBA,RS305LBA,RS306LBA,RS406LBA,RS407LBA,RS503LBA]] \ sol.applycal.parmdb=ddcal/solutions/cal-core-c' +str(c)+'.h5 sol.applycal.correction=tec000 \ sol.solint=10 sol.nchan=4 sol.h5parm=$pathMS/cal-remote-c' +str(c)+'.h5 sol.sourcedb='+skymodel_cl_skydb, \ log='$nameMS_solTECremote-c'+str(c)+'.log', commandType='DPPP') # Plot solutions
for c in range(3): logger.info(f'Cycle {c}: {threads[c]} thread(s) @ {cpu_per_thread[c]} cpu') s = lib_util.Scheduler(log_dir=logger_obj.log_dir, dry=False, maxThreads=threads[c]) # solve TEC - ms:SMOOTHED_DATA logger.info('Solving TEC...') start = time.time() MSs.run('DPPP ' + parset_dir + '/DPPP-solTEC.parset msin=$pathMS sol.h5parm=$pathMS/tec.h5 \ sol.antennaconstraint=[[CS001LBA,CS002LBA,CS003LBA,CS004LBA,CS005LBA,CS006LBA,CS007LBA]]', log=f'$nameMS_solTEC-c{c}.log', commandType='DPPP') #maxThreads logger.info(f'DP3: {start-time.time()}') lib_util.run_losoto(s, 'tec-c' + str(c), [ms + '/tec.h5' for ms in MSs.getListStr()], [parset_dir + '/losoto-plot-tec.parset']) logger.info(f'DP3 + LoSoTo: {start-time.time()}') os.system('mv cal-tec-c' + str(c) + '.h5 bench/') os.system('mv plots-tec-c' + str(c) + ' bench/plots/') sys.exit() ##################################################################################################### # Self-cal cycle for c in range(2): logger.info('Start selfcal cycle: ' + str(c)) if c == 0: with w.if_todo('set_corrected_data'): logger.info('Set CORRECTED_DATA = DATA...') MSs.run('taql "update $pathMS set CORRECTED_DATA = DATA"',
# Smoothing - ms:SUBTRACTED_DATA -> ms:SMOOTHED_DATA logger.info('BL-based smoothing...') MSs.run( 'BLsmooth.py -f 1.0 -r -i SUBTRACTED_DATA -o SMOOTHED_DATA $pathMS', log='$nameMS_smooth-c' + str(c) + '.log', commandType='python') # Calibration - ms:SMOOTHED_DATA logger.info('Calibrating...') MSs.run('DPPP '+parset_dir+'/DPPP-solDD.parset msin=$pathMS ddecal.h5parm=$pathMS/cal-c'+str(c)+'.h5 ddecal.sourcedb='+skymodel_cl_skydb, \ log='$nameMS_solDD-c'+str(c)+'.log', commandType='DPPP') # Plot solutions lib_util.run_losoto( s, 'c' + str(c), [MS + '/cal-c' + str(c) + '.h5' for MS in MSs.getListStr()], [parset_dir + '/losoto-plot.parset']) os.system('mv plots-c' + str(c) + '* ddcal/plots') ############################################################## # low S/N DIE corrections # TODO: add a higher low-uv cut to remove galaxy effect if c >= 0: logger.info('DIE calibration...') # predict and corrupt each facet logger.info('Reset MODEL_DATA...') MSs.run('taql "update $pathMS set MODEL_DATA = 0"', log='$nameMS_taql-c' + str(c) + '.log', commandType='general') for i, d in enumerate(directions):