def simsky(msname="$MS", lsmname="$LSM", column="$COLUMN", tdlconf="$TDLCONF", tdlsec="$SIMSEC", addnoise=True, noise=0, sefd=0, recenter=True, options={} ,args=[],**kw): """ Simulates visibilities into a MS. msname : MS name lsmname : LSM name column : Column to simulate visibilities into tdlconf : Meqtrees TDL configuration profiles file (required to run MeqTrees pipeliner) tdlsec : Section to execute in tdlconf noise : Visibility noise to add to simulation. args, kw : extra arguments to pass to the MeqTrees pipeliner """ msname,lsmname,column,tdlsec,tdlconf = interpolate_locals('msname lsmname column tdlsec tdlconf') # recenter LSM if required if recenter: x.sh('tigger-convert --recenter=$DIRECTION $lsmname $RLSM -f') v.LSM = RLSM else: v.LSM = lsmname args = ["${ms.MS_TDL} ${lsm.LSM_TDL}"] + list(args) options["pybeams_fits.filename_pattern"] = BEAM_PATTERN_HOL options['ms_sel.output_column'] = column if addnoise: sefd = sefd or SEFD options['noise_stddev'] = noise or compute_vis_noise(sefd) options.update(kw) # extra keyword args get preference mqt.run(TURBO_SIM,job='_tdl_job_1_simulate_MS',config=tdlconf,section=tdlsec,options=options,args=args)
def simsky(msname='$MS', lsmname='$LSM', tdlsec='$TDLSEC', tdlconf='$TDLCONF', column='$COLUMN', noise=0, args=[], addToCol=None, **kw): """ Simulates visibilities into an MS """ msname, lsmname, column, tdlsec, tdlconf = interpolate_locals( 'msname lsmname' ' column tdlsec tdlconf') fits = True if verify_sky(lsmname) is 'FITS' else False v.MS = msname v.LSM = lsmname _column = 'MODEL_DATA' if addToCol else column if fits: _column = 'MODEL_DATA' if noise else column im.argo.predict_vis(lsmname, wprojplanes=128, column=_column) if noise: simnoise(noise=noise, addToCol=_column, column=column) else: args = ["${ms.MS_TDL} ${lsm.LSM_TDL}"] + list(args) options = {} options['ms_sel.output_column'] = _column if noise: options['noise_stddev'] = noise mqt.run(TURBO_SIM, job='_tdl_job_1_simulate_MS', config=tdlconf, section=tdlsec, options=options, args=args, **kw) if addToCol: tab = ms.msw() col1 = tab.getcol(addToCol) col2 = tab.getcol('MODEL_DATA') comb = col1 + col2 nrows = len(comb) rowchunk = nrows // 5 for row0 in range(0, nrows, rowchunk): nr = min(nrows - row0, rowchunk) info('MODEL_DATA + $addToCol --> $column : rows %d-%d' % (row0, row0 + nr)) tab.putcol(column, comb[row0:row0 + nr], row0, nr) tab.close()
def addnoise (noise=0,rowchunk=100000): """adds noise to MODEL_DATA, writes to CORRECTED_DATA"""; # compute expected noise noise = compute_vis_noise(noise); # fill MS with noise # setup stefcal options and run info("Running turbo-sim to add noise to data"); # setup args args = [ """${ms.MS_TDL} ${ms.CHAN_TDL} ms_sel.ms_ifr_subset_str=${ms.IFRS} noise_stddev=%g"""%noise ]; mqt.run("${mqt.CATTERY}/Siamese/turbo-sim.py","simulate",section="addnoise",args=args);
def cal_function(type_cal="LM"): options = {} options['ms_sel.msname']=II("${MS}") options['ms_sel.tile_size']=get_time_slots() v.CALORNOT = "cal_model" #options['ms_sel.output_column']="CORRECTED_DATA" options['tiggerlsm.filename']=II("${LSM}") if type_cal=="LM": mqt.run(script="calico-wsrt-tens.py",job="cal_G_diag",config="tdlconf.profiles",section="selfcal",options=options) elif type_cal=="STEF": mqt.run(script="calico-stefcal.py",job="stefcal",config="tdlconf.profiles",section="G_calibration",options=options) #elif type_cal=="STEF_DIR": #CALLING STEFCAL directly fails at the moment # stefcal.stefcal(section="G_calibration",output="CORR_DATA",options=options) v.CALORNOT = ''
def sim_function(cal=False,whole=False): options = {} options['ms_sel.msname']=II("${MS}") options['ms_sel.tile_size']=get_time_slots() if cal: if not whole: v.CALORNOT = "cal_model" options['ms_sel.output_column']="DATA" else: options['ms_sel.output_column']="CORRECTED_DATA" options['tiggerlsm.filename']=II("${LSM}") mqt.run(script="turbo-sim.py",job="_tdl_job_1_simulate_MS",config="tdlconf.profiles",section="sim",options=options) v.CALORNOT = ''
def addnoise(noise=0, rowchunk=100000): """adds noise to MODEL_DATA, writes to CORRECTED_DATA""" # compute expected noise noise = compute_vis_noise(noise) # fill MS with noise # setup stefcal options and run info("Running turbo-sim to add noise to data") # setup args args = [ """${ms.MS_TDL} ${ms.CHAN_TDL} ms_sel.ms_ifr_subset_str=${ms.IFRS} noise_stddev=%g""" % noise ] mqt.run("${mqt.CATTERY}/Siamese/turbo-sim.py", "simulate", section="addnoise", args=args)
def simsky(msname="$MS", lsmname="$LSM", column="$COLUMN", tdlconf="$TDLCONF", tdlsec="$SIMSEC", addnoise=True, noise=0, sefd=0, recenter=True, options={}, args=[], **kw): """ Simulates visibilities into a MS. msname : MS name lsmname : LSM name column : Column to simulate visibilities into tdlconf : Meqtrees TDL configuration profiles file (required to run MeqTrees pipeliner) tdlsec : Section to execute in tdlconf noise : Visibility noise to add to simulation. args, kw : extra arguments to pass to the MeqTrees pipeliner """ msname, lsmname, column, tdlsec, tdlconf = interpolate_locals( 'msname lsmname column tdlsec tdlconf') # recenter LSM if required if recenter: x.sh('tigger-convert --recenter=$DIRECTION $lsmname $RLSM -f') v.LSM = RLSM else: v.LSM = lsmname args = ["${ms.MS_TDL} ${lsm.LSM_TDL}"] + list(args) options["pybeams_fits.filename_pattern"] = BEAM_PATTERN_HOL options['ms_sel.output_column'] = column if addnoise: sefd = sefd or SEFD options['noise_stddev'] = noise or compute_vis_noise(sefd) options.update(kw) # extra keyword args get preference mqt.run(TURBO_SIM, job='_tdl_job_1_simulate_MS', config=tdlconf, section=tdlsec, options=options, args=args)
def simsky(msname='$MS', lsmname='$LSM', tdlsec='$TDLSEC', tdlconf='$TDLCONF', column='$COLUMN', noise=0, args=[], addToCol=None,**kw): """ Simulates visibilities into an MS """ msname, lsmname, column, tdlsec, tdlconf = interpolate_locals('msname lsmname' ' column tdlsec tdlconf') fits = True if verify_sky(lsmname) is 'FITS' else False v.MS = msname v.LSM = lsmname _column = 'MODEL_DATA' if addToCol else column if fits: _column = 'MODEL_DATA' if noise else column im.argo.predict_vis(lsmname, wprojplanes=128, column=_column) if noise: simnoise(noise=noise,addToCol=_column,column=column) else: args = ["${ms.MS_TDL} ${lsm.LSM_TDL}"] + list(args) options = {} options['ms_sel.output_column'] = _column if noise: options['noise_stddev'] = noise mqt.run(TURBO_SIM, job='_tdl_job_1_simulate_MS', config=tdlconf, section=tdlsec, options=options, args=args,**kw) if addToCol: tab = ms.msw() col1 = tab.getcol(addToCol) col2 = tab.getcol('MODEL_DATA') comb = col1 + col2 nrows = len(comb) rowchunk = nrows//5 for row0 in range(0,nrows,rowchunk): nr = min(nrows-row0,rowchunk) info('MODEL_DATA + $addToCol --> $column : rows %d-%d'%(row0,row0+nr) ) tab.putcol(column,comb[row0:row0+nr],row0,nr) tab.close()
def add_pjones(output_column): options = {} options['ms_sel.msname'] = II('$MS') options['ms_sel.output_column'] = output_column options['read_ms_model'] = 1 # read existing visibilities from MS options['ms_sel.model_column'] = output_column options['ms_sel.tile_size'] = 1000000 options['sim_mode'] = 'sim only' options['me.p_enable'] = 1 options['feed_angle.enable_pa'] = 1 # enable parallactic angle rotation options[ 'feed_angle.read_ms'] = 1 # enable reading feed angle from FEED subtable in MS mqt.MULTITHREAD = 32 #max number of meqserver threads mqt.run(script=II('$FRAMEWORKDIR') + '/turbo-sim.py', config=II('$FRAMEWORKDIR') + '/tdlconf.profiles', job='_simulate_MS', options=options)
def simulate(msname='$MS', lsmname='$LSM', tdlconf='$TDLCONF', section='$TDLSEC', freq0=1400e6, options={}, **kw): """ Simulates visibilities into an MS """ v.MS, v.LSM = interpolate_locals('msname lsmname') if LSM: options['tiggerlsm.filename'] = LSM options['noise_stddev'] = NOISE or compute_vis_noise(sefd=get_sefd(freq0)) options['ms_sel.output_column'] = COLUMN if USING_SIAMESE: section = 'turbo-sim:default' options['ms_sel.select_channels'] = 0 mqt.run('turbo-sim.py', job='_tdl_job_1_simulate_MS', config=tdlconf, section=section, options=options)
def sim_now(): radius1 = np.arange(0., 100., 10.) #radius1 = np.arange(0.,3,0.3) Flux_array1 = np.zeros(len(radius1)) lores1 = "MeerKATlores.MS_p0_p0" imager.cellsize = "10.arcsec" imager.npix = 512 imager.niter = 0 imager.CLEAN_ALGORITHM = "csclean" for k in range(len(radius1)): options = {} options['gridded_sky.grid_m0'] = radius1[k] options['ms_sel.msname'] = lores1 mqt.run("turbo-sim.py", job="_tdl_job_1_simulate_MS", config="tdlconf.profiles1", section="Sim_source_radius", options=options) center_min = -45 * 60 + radius1[k] center_deg = math.ceil(center_min / 60) center_min = abs(center_min - center_deg * 60) imager.make_image(msname=lores1, column='CORRECTED_DATA', phasecenter="j2000,0h0m,%dd%dm" % (center_deg, center_min), restore=False, dirty=True, restore_lsm=False, weight="natural") f1 = np.max(pyfits.open(imager.DIRTY_IMAGE)[0].data[0][0]) Flux_array1[k] = f1 np.save("DATA/Flux1", Flux_array1) np.save("DATA/radius1", radius1)
def run_turbosim(input_fitsimage, output_column, taql_string): options = {} options['ms_sel.msname'] = II('$MS') options['ms_sel.output_column'] = output_column if input_fitsimage.endswith(('.fits', '.FITS')): options['me.sky.siamese_oms_fitsimage_sky'] = 1 options['fitsimage_sky.image_filename'] = input_fitsimage options['fitsimage_sky.pad_factor'] = 2.4 elif input_fitsimage.endswith(('.txt', '.html')): options['me.sky.tiggerskymodel'] = 1 options['tiggerlsm.filename'] = input_fitsimage options['ms_sel.tile_size'] = 1000000 options['ms_sel.ms_taql_str'] = taql_string mqt.MULTITHREAD = 32 #max number of meqserver threads mqt.run(script=II('$FRAMEWORKDIR') + '/turbo-sim.py', config=II('$FRAMEWORKDIR') + '/tdlconf.profiles', section='turbo-sim', job='_simulate_MS', options=options)
def stefcal ( msname="$MS",section="$STEFCAL_SECTION", diffgains=None, apply_only=False, gain_apply_only=False, diffgain_apply_only=False, ifrgain_apply_only=False, diffgain_intervals=None,diffgain_smoothing=None, flag_threshold=None, output="CORR_RES", plotvis="${ms.PLOTVIS}", dirty=True,restore=False,restore_lsm=True, label=None, args=[],options={}, **kws): """Generic function to run a stefcal job. 'section' TDL config file section 'label' will be assigned to the global LABEL for purposes of file naming 'apply_only' if true, will only apply saved solutions 'diffgains' set to a source subset string to solve for diffgains. Set to True to use "=dE" 'diffgain_mode' 'solve-save' to solve & save, 'solve-nosave' to not save, 'apply' to apply only 'flag_threshold' threshold flaging post-solutions. Give one threshold to flag with --above, or T1,T2 for --above T1 --fm-above T2 'output' output visibilities ('CORR_DATA','CORR_RES', 'RES' are useful) 'plotvis' if not empty, specifies which output visibilities to plot using plot-ms (see plot.ms.py --help) 'dirty','restore' 'restore_lsm' image output visibilities (passed to imager.make_image above as is) 'args','options' passed to the stefcal job as is (as a list of arguments and kw=value pairs), can be used to supply extra TDL options extra keywords: passed to the stefcal job as kw=value, can be used to supply extra TDL options, thus overriding settings in the TDL config file. Useful arguments of this kind are e.g.: stefcal_reset_all=True to remove prior gains solutions. """ msname,section,lsm,label,plotvis = interpolate_locals("msname section lsm label plotvis"); makedir(v.DESTDIR); # increment step counter and assign global label if label is not None: v.LABEL = str(label); if type(v.STEP) is int and STEFCAL_STEP_INCR: v.STEP += STEFCAL_STEP_INCR; # setup stefcal options and run info("Running stefcal ${step <STEP} ${(<LABEL>)}"); # setup args args0 = [ """${ms.MS_TDL} ${ms.CHAN_TDL} ${lsm.LSM_TDL} ms_sel.ms_ifr_subset_str=${ms.IFRS} ms_sel.output_column=$STEFCAL_OUTPUT_COLUMN stefcal_gain.enabled=1 stefcal_diffgain.enabled=%d %s"""% ((1 if diffgains else 0),STEFCAL_TDLOPTS) ]; if diffgains: if diffgains is True: diffgains = "=dE"; args0.append("de_subset.subset_enabled=1 de_subset.source_subset=$diffgains"); opts = { 'do_output': output, 'stefcal_gain.mode': "apply" if apply_only or gain_apply_only else "solve-save", 'stefcal_gain1.mode': "apply" if apply_only or gain_apply_only else "solve-save", 'stefcal_diffgain.mode': "apply" if apply_only or diffgain_apply_only else "solve-save", 'stefcal_diffgain1.mode': "apply" if apply_only or diffgain_apply_only else "solve-save", 'stefcal_ifr_gain_mode': "apply" if apply_only or ifrgain_apply_only else "solve-save", 'stefcal_gain.table': STEFCAL_GAIN, 'stefcal_gain1.table': STEFCAL_GAIN1, 'stefcal_diffgain.table': STEFCAL_DIFFGAIN, 'stefcal_diffgain1.table': STEFCAL_DIFFGAIN1, 'stefcal_ifr_gain_table': STEFCAL_IFRGAIN, 'stefcal_visualize': False } timesmooth,freqsmooth = diffgain_smoothing or STEFCAL_DIFFGAIN_SMOOTHING or (0,0); timeint,freqint = diffgain_intervals or STEFCAL_DIFFGAIN_INTERVALS or (0,0); opts['stefcal_diffgain.timeint'] = 0 if timesmooth else timeint; opts['stefcal_diffgain.freqint'] = 0 if freqsmooth else freqint; opts['stefcal_diffgain.timesmooth'] = timesmooth; opts['stefcal_diffgain.freqsmooth'] = freqsmooth; # add user-defined args args0 += list(args); opts.update(options); opts.update(kws); # run the job mqt.run(STEFCAL_SCRIPT,STEFCAL_JOBNAME,section=section,args=args0,options=opts); # copy gains if not apply_only: if os.path.exists(STEFCAL_GAIN): std.copy(STEFCAL_GAIN,STEFCAL_GAIN_SAVE); if os.path.exists(STEFCAL_GAIN1): std.copy(STEFCAL_GAIN1,STEFCAL_GAIN1_SAVE); if os.path.exists(STEFCAL_DIFFGAIN): std.copy(STEFCAL_DIFFGAIN,STEFCAL_DIFFGAIN_SAVE); if os.path.exists(STEFCAL_IFRGAIN): std.copy(STEFCAL_IFRGAIN,STEFCAL_IFRGAIN_SAVE); # post-calibration flagging if flag_threshold: if isinstance(flag_threshold,(list,tuple)): t0,t1 = flag_threshold; else: t0,t1 = flag_threshold,None; ms.flagms("--above %g"%t0,"-f threshold -c"); if t1: ms.flagms("--fm-above %g"%t1,"-f fmthreshold -c"); # plot residuals if plotvis: info("Plotting visibilities ($plotvis)"); ms.PLOTVIS = plotvis; ms.plotms("-o ${OUTFILE}_${output}${_s<STEP}${_<label}.png"); # make images imager.make_image(msname,column=STEFCAL_OUTPUT_COLUMN,dirty=dirty,restore=restore,restore_lsm=restore_lsm);
def stefcal(msname="$MS", section="$STEFCAL_SECTION", diffgains=None, apply_only=False, reset=False, gain_apply_only=False, gain_reset=False, diffgain_apply_only=False, diffgain_reset=False, gain_plot_prefix="$STEFCAL_GAIN_PLOT_PREFIX", gain1_plot_prefix="$STEFCAL_GAIN1_PLOT_PREFIX", ifrgain_plot_prefix="$STEFCAL_IFRGAIN_PLOT_PREFIX", diffgain_plot_prefix="$STEFCAL_DIFFGAIN_PLOT_PREFIX", ifrgain_apply_only=False, ifrgain_reset=False, gain_intervals=None, gain_smoothing=None, diffgain_intervals=None, diffgain_smoothing=None, flag_threshold=None, calibrate_ifrs="$STEFCAL_CALIBRATE_IFRS", input_column="$STEFCAL_INPUT_COLUMN", output_column="$STEFCAL_OUTPUT_COLUMN", output="CORR_RES", plotvis="${ms.PLOTVIS}", dirty=True, restore=False, restore_lsm=True, label=None, saveconfig="$STEFCAL_SAVE_CONFIG", plotfail=None, args=[], options={}, **kws): """Generic function to run a stefcal job. 'section' TDL config file section 'label' will be assigned to the global LABEL for purposes of file naming 'apply_only' if true, will only apply saved solutions rather than re-solve '{gain,diffgain,ifrgain}_apply_only' if true, will only apply saved gain/diffgain/IFR gain solutions rather than re-solve 'reset' if true, will reset all saved solutuions prior to starting '{gain,diffgain,ifrgain}_reset' if true, will reset saved gain/diffgain/IFR gain solutuions prior to starting 'diffgains' set to a source subset string to solve for diffgains. Set to True to use "=dE" 'diffgain_mode' 'solve-save' to solve & save, 'solve-nosave' to not save, 'apply' to apply only 'diffgain_plot' automatically invoke make_diffgain_plots() if True 'flag_threshold' threshold flaging post-solutions. Give one threshold to flag with --above, or T1,T2 for --above T1 --fm-above T2 'output' output visibilities ('CORR_DATA','CORR_RES', 'RES' are useful) 'plotvis' if not empty, specifies which output visibilities to plot using plot-ms (see plot.ms.py --help) 'dirty','restore' 'restore_lsm' image output visibilities (passed to imager.make_image above as is) 'plotfail' plotting failure reported via warn or abort. Default is warn, set to abort to abort. 'saveconfig' saves the effective TDL config to file[:section] 'args','options' passed to the stefcal job as is (as a list of arguments and kw=value pairs), can be used to supply extra TDL options extra keywords: passed to the stefcal job as kw=value, can be used to supply extra TDL options, thus overriding settings in the TDL config file. Useful arguments of this kind are e.g.: stefcal_reset_all=True to remove prior gains solutions. """ msname,section,lsm,label,plotvis,calibrate_ifrs, \ gain_plot_prefix,gain1_plot_prefix,ifrgain_plot_prefix,diffgain_plot_prefix,saveconfig,plotfail,input_column,output_column = \ interpolate_locals("msname section lsm label plotvis calibrate_ifrs gain_plot_prefix gain1_plot_prefix ifrgain_plot_prefix diffgain_plot_prefix " "saveconfig plotfail input_column output_column") plotfail = plotfail or warn makedir(v.DESTDIR) # increment step counter and assign global label if label is not None: v.LABEL = str(label) if type(v.STEP) is int and STEFCAL_STEP_INCR: v.STEP += STEFCAL_STEP_INCR # setup stefcal options and run info("Running stefcal ${step <STEP} ${(<LABEL>)}") # setup args args0 = [ """${ms.MS_TDL} ${ms.CHAN_TDL} ${lsm.LSM_TDL} ms_sel.ms_ifr_subset_str=${ms.IFRS} ms_sel.input_column=$input_column ms_sel.output_column=$output_column stefcal_gain.enabled=1 stefcal_diffgain.enabled=%d %s""" % ((1 if diffgains else 0), STEFCAL_TDLOPTS) ] if diffgains: if diffgains is True: diffgains = "=dE" args0.append( "de_subset.subset_enabled=1 de_subset.source_subset=$diffgains") opts = { 'do_output': output, 'calibrate_ifrs': calibrate_ifrs, 'stefcal_gain.mode': "apply" if apply_only or gain_apply_only else "solve-save", 'stefcal_gain1.mode': "apply" if apply_only or gain_apply_only else "solve-save", 'stefcal_gain.reset': int(reset or gain_reset), 'stefcal_diffgain.mode': "apply" if apply_only or diffgain_apply_only else "solve-save", 'stefcal_diffgain1.mode': "apply" if apply_only or diffgain_apply_only else "solve-save", 'stefcal_diffgain.reset': int(reset or diffgain_reset), 'stefcal_diffgain1.reset': int(reset or diffgain_reset), 'stefcal_ifr_gain_mode': "apply" if apply_only or ifrgain_apply_only else "solve-save", 'stefcal_ifr_gain_reset': int(reset or ifrgain_reset), 'stefcal_gain.table': STEFCAL_GAIN, 'stefcal_gain1.table': STEFCAL_GAIN1, 'stefcal_diffgain.table': STEFCAL_DIFFGAIN, 'stefcal_diffgain1.table': STEFCAL_DIFFGAIN1, 'stefcal_ifr_gain_table': STEFCAL_IFRGAIN, 'stefcal_visualize': False } # set gain parameters if gain_smoothing or STEFCAL_GAIN_SMOOTHING or gain_intervals or STEFCAL_GAIN_INTERVALS: timesmooth, freqsmooth = gain_smoothing or STEFCAL_GAIN_SMOOTHING or ( 0, 0) timeint, freqint = gain_intervals or STEFCAL_GAIN_INTERVALS or (0, 0) opts['stefcal_gain.timeint'] = 0 if timesmooth else timeint opts['stefcal_gain.freqint'] = 0 if freqsmooth else freqint opts['stefcal_gain.timesmooth'] = timesmooth opts['stefcal_gain.freqsmooth'] = freqsmooth # set diffgain parameters if diffgain_smoothing or STEFCAL_DIFFGAIN_SMOOTHING or diffgain_intervals or STEFCAL_DIFFGAIN_INTERVALS: timesmooth, freqsmooth = diffgain_smoothing or STEFCAL_DIFFGAIN_SMOOTHING or ( 0, 0) timeint, freqint = diffgain_intervals or STEFCAL_DIFFGAIN_INTERVALS or ( 0, 0) opts['stefcal_diffgain.timeint'] = 0 if timesmooth else timeint opts['stefcal_diffgain.freqint'] = 0 if freqsmooth else freqint opts['stefcal_diffgain.timesmooth'] = timesmooth opts['stefcal_diffgain.freqsmooth'] = freqsmooth # add user-defined args args0 += list(args) opts.update(options) opts.update(kws) # run the job mqt.run(STEFCAL_SCRIPT, STEFCAL_JOBNAME, section=section, saveconfig=saveconfig, args=args0, options=opts) # copy gains try: if not apply_only: if os.path.exists(STEFCAL_GAIN) and not gain_apply_only: std.copy(STEFCAL_GAIN, STEFCAL_GAIN_SAVE) if gain_plot_prefix: make_gain_plots(STEFCAL_GAIN_SAVE, prefix=gain_plot_prefix) if os.path.exists(STEFCAL_GAIN1) and not gain_apply_only: std.copy(STEFCAL_GAIN1, STEFCAL_GAIN1_SAVE) if gain_plot_prefix: make_gain_plots(STEFCAL_GAIN1_SAVE, prefix=gain_plot_prefix) if os.path.exists(STEFCAL_DIFFGAIN) and not diffgain_apply_only: std.copy(STEFCAL_DIFFGAIN, STEFCAL_DIFFGAIN_SAVE) if diffgain_plot_prefix: make_diffgain_plots(STEFCAL_DIFFGAIN_SAVE, prefix=diffgain_plot_prefix) if os.path.exists(STEFCAL_IFRGAIN) and not ifrgain_apply_only: std.copy(STEFCAL_IFRGAIN, STEFCAL_IFRGAIN_SAVE) if ifrgain_plot_prefix: make_ifrgain_plots(STEFCAL_IFRGAIN_SAVE, prefix=ifrgain_plot_prefix) except: traceback.print_exc() plotfail("plot routine failed, see exception above") # post-calibration flagging if flag_threshold: if isinstance(flag_threshold, (list, tuple)): t0, t1 = flag_threshold else: t0, t1 = flag_threshold, None ms.flagms("--above %g" % t0, "-f threshold -c") if t1: ms.flagms("--fm-above %g" % t1, "-f fmthreshold -c") # plot residuals if plotvis: try: info("Plotting visibilities ($plotvis)") ms.PLOTVIS = plotvis ms.plotms("-o ${OUTFILE}_${output}${_s<STEP}${_<label}.png") except: traceback.print_exc() plotfail("plot routine failed, see exception above") # make images im.make_image(msname, column=STEFCAL_OUTPUT_COLUMN, dirty=dirty, restore=restore, restore_lsm=restore_lsm)
def stefcal(msname="$MS", section="$STEFCAL_SECTION", diffgains=None, apply_only=False, gain_apply_only=False, diffgain_apply_only=False, ifrgain_apply_only=False, diffgain_intervals=None, diffgain_smoothing=None, flag_threshold=None, output="CORR_RES", plotvis="${ms.PLOTVIS}", dirty=True, restore=False, restore_lsm=True, label=None, args=[], options={}, **kws): """Generic function to run a stefcal job. 'section' TDL config file section 'label' will be assigned to the global LABEL for purposes of file naming 'apply_only' if true, will only apply saved solutions 'diffgains' set to a source subset string to solve for diffgains. Set to True to use "=dE" 'diffgain_mode' 'solve-save' to solve & save, 'solve-nosave' to not save, 'apply' to apply only 'flag_threshold' threshold flaging post-solutions. Give one threshold to flag with --above, or T1,T2 for --above T1 --fm-above T2 'output' output visibilities ('CORR_DATA','CORR_RES', 'RES' are useful) 'plotvis' if not empty, specifies which output visibilities to plot using plot-ms (see plot.ms.py --help) 'dirty','restore' 'restore_lsm' image output visibilities (passed to imager.make_image above as is) 'args','options' passed to the stefcal job as is (as a list of arguments and kw=value pairs), can be used to supply extra TDL options extra keywords: passed to the stefcal job as kw=value, can be used to supply extra TDL options, thus overriding settings in the TDL config file. Useful arguments of this kind are e.g.: stefcal_reset_all=True to remove prior gains solutions. """ msname, section, lsm, label, plotvis = interpolate_locals( "msname section lsm label plotvis") makedir(v.DESTDIR) # increment step counter and assign global label if label is not None: v.LABEL = str(label) if type(v.STEP) is int and STEFCAL_STEP_INCR: v.STEP += STEFCAL_STEP_INCR # setup stefcal options and run info("Running stefcal ${step <STEP} ${(<LABEL>)}") # setup args args0 = [ """${ms.MS_TDL} ${ms.CHAN_TDL} ${lsm.LSM_TDL} ms_sel.ms_ifr_subset_str=${ms.IFRS} ms_sel.output_column=$STEFCAL_OUTPUT_COLUMN stefcal_gain.enabled=1 stefcal_diffgain.enabled=%d %s""" % ((1 if diffgains else 0), STEFCAL_TDLOPTS) ] if diffgains: if diffgains is True: diffgains = "=dE" args0.append( "de_subset.subset_enabled=1 de_subset.source_subset=$diffgains") opts = { 'do_output': output, 'stefcal_gain.mode': "apply" if apply_only or gain_apply_only else "solve-save", 'stefcal_gain1.mode': "apply" if apply_only or gain_apply_only else "solve-save", 'stefcal_diffgain.mode': "apply" if apply_only or diffgain_apply_only else "solve-save", 'stefcal_diffgain1.mode': "apply" if apply_only or diffgain_apply_only else "solve-save", 'stefcal_ifr_gain_mode': "apply" if apply_only or ifrgain_apply_only else "solve-save", 'stefcal_gain.table': STEFCAL_GAIN, 'stefcal_gain1.table': STEFCAL_GAIN1, 'stefcal_diffgain.table': STEFCAL_DIFFGAIN, 'stefcal_diffgain1.table': STEFCAL_DIFFGAIN1, 'stefcal_ifr_gain_table': STEFCAL_IFRGAIN, 'stefcal_visualize': False } timesmooth, freqsmooth = diffgain_smoothing or STEFCAL_DIFFGAIN_SMOOTHING or ( 0, 0) timeint, freqint = diffgain_intervals or STEFCAL_DIFFGAIN_INTERVALS or (0, 0) opts['stefcal_diffgain.timeint'] = 0 if timesmooth else timeint opts['stefcal_diffgain.freqint'] = 0 if freqsmooth else freqint opts['stefcal_diffgain.timesmooth'] = timesmooth opts['stefcal_diffgain.freqsmooth'] = freqsmooth # add user-defined args args0 += list(args) opts.update(options) opts.update(kws) # run the job mqt.run(STEFCAL_SCRIPT, STEFCAL_JOBNAME, section=section, args=args0, options=opts) # copy gains if not apply_only: if os.path.exists(STEFCAL_GAIN): std.copy(STEFCAL_GAIN, STEFCAL_GAIN_SAVE) if os.path.exists(STEFCAL_GAIN1): std.copy(STEFCAL_GAIN1, STEFCAL_GAIN1_SAVE) if os.path.exists(STEFCAL_DIFFGAIN): std.copy(STEFCAL_DIFFGAIN, STEFCAL_DIFFGAIN_SAVE) if os.path.exists(STEFCAL_IFRGAIN): std.copy(STEFCAL_IFRGAIN, STEFCAL_IFRGAIN_SAVE) # post-calibration flagging if flag_threshold: if isinstance(flag_threshold, (list, tuple)): t0, t1 = flag_threshold else: t0, t1 = flag_threshold, None ms.flagms("--above %g" % t0, "-f threshold -c") if t1: ms.flagms("--fm-above %g" % t1, "-f fmthreshold -c") # plot residuals if plotvis: info("Plotting visibilities ($plotvis)") ms.PLOTVIS = plotvis ms.plotms("-o ${OUTFILE}_${output}${_s<STEP}${_<label}.png") # make images imager.make_image(msname, column=STEFCAL_OUTPUT_COLUMN, dirty=dirty, restore=restore, restore_lsm=restore_lsm)
def stefcal ( msname="$MS",section="$STEFCAL_SECTION", diffgains=None, apply_only=False, reset=False, gain_apply_only=False, gain_reset=False, diffgain_apply_only=False, diffgain_reset=False, gain_plot_prefix="$STEFCAL_GAIN_PLOT_PREFIX", gain1_plot_prefix="$STEFCAL_GAIN1_PLOT_PREFIX", ifrgain_plot_prefix="$STEFCAL_IFRGAIN_PLOT_PREFIX", diffgain_plot_prefix="$STEFCAL_DIFFGAIN_PLOT_PREFIX", ifrgain_apply_only=False, ifrgain_reset=False, gain_intervals=None,gain_smoothing=None, diffgain_intervals=None,diffgain_smoothing=None, flag_threshold=None, calibrate_ifrs="$STEFCAL_CALIBRATE_IFRS", input_column="$STEFCAL_INPUT_COLUMN", output_column="$STEFCAL_OUTPUT_COLUMN", output="CORR_RES", plotvis="${ms.PLOTVIS}", dirty=True,restore=False,restore_lsm=True, label=None, saveconfig="$STEFCAL_SAVE_CONFIG", plotfail=None, args=[],options={}, **kws): """Generic function to run a stefcal job. 'section' TDL config file section 'label' will be assigned to the global LABEL for purposes of file naming 'apply_only' if true, will only apply saved solutions rather than re-solve '{gain,diffgain,ifrgain}_apply_only' if true, will only apply saved gain/diffgain/IFR gain solutions rather than re-solve 'reset' if true, will reset all saved solutuions prior to starting '{gain,diffgain,ifrgain}_reset' if true, will reset saved gain/diffgain/IFR gain solutuions prior to starting 'diffgains' set to a source subset string to solve for diffgains. Set to True to use "=dE" 'diffgain_mode' 'solve-save' to solve & save, 'solve-nosave' to not save, 'apply' to apply only 'diffgain_plot' automatically invoke make_diffgain_plots() if True 'flag_threshold' threshold flaging post-solutions. Give one threshold to flag with --above, or T1,T2 for --above T1 --fm-above T2 'output' output visibilities ('CORR_DATA','CORR_RES', 'RES' are useful) 'plotvis' if not empty, specifies which output visibilities to plot using plot-ms (see plot.ms.py --help) 'dirty','restore' 'restore_lsm' image output visibilities (passed to imager.make_image above as is) 'plotfail' plotting failure reported via warn or abort. Default is warn, set to abort to abort. 'saveconfig' saves the effective TDL config to file[:section] 'args','options' passed to the stefcal job as is (as a list of arguments and kw=value pairs), can be used to supply extra TDL options extra keywords: passed to the stefcal job as kw=value, can be used to supply extra TDL options, thus overriding settings in the TDL config file. Useful arguments of this kind are e.g.: stefcal_reset_all=True to remove prior gains solutions. """ msname,section,lsm,label,plotvis,calibrate_ifrs, \ gain_plot_prefix,gain1_plot_prefix,ifrgain_plot_prefix,diffgain_plot_prefix,saveconfig,plotfail,input_column,output_column = \ interpolate_locals("msname section lsm label plotvis calibrate_ifrs gain_plot_prefix gain1_plot_prefix ifrgain_plot_prefix diffgain_plot_prefix " "saveconfig plotfail input_column output_column") plotfail = plotfail or warn makedir(v.DESTDIR); # increment step counter and assign global label if label is not None: v.LABEL = str(label); if type(v.STEP) is int and STEFCAL_STEP_INCR: v.STEP += STEFCAL_STEP_INCR; # setup stefcal options and run info("Running stefcal ${step <STEP} ${(<LABEL>)}"); # setup args args0 = [ """${ms.MS_TDL} ${ms.CHAN_TDL} ${lsm.LSM_TDL} ms_sel.ms_ifr_subset_str=${ms.IFRS} ms_sel.input_column=$input_column ms_sel.output_column=$output_column stefcal_gain.enabled=1 stefcal_diffgain.enabled=%d %s"""% ((1 if diffgains else 0),STEFCAL_TDLOPTS) ]; if diffgains: if diffgains is True: diffgains = "=dE"; args0.append("de_subset.subset_enabled=1 de_subset.source_subset=$diffgains"); opts = { 'do_output': output, 'calibrate_ifrs': calibrate_ifrs, 'stefcal_gain.mode': "apply" if apply_only or gain_apply_only else "solve-save", 'stefcal_gain1.mode': "apply" if apply_only or gain_apply_only else "solve-save", 'stefcal_gain.reset': int(reset or gain_reset), 'stefcal_diffgain.mode': "apply" if apply_only or diffgain_apply_only else "solve-save", 'stefcal_diffgain1.mode': "apply" if apply_only or diffgain_apply_only else "solve-save", 'stefcal_diffgain.reset': int(reset or diffgain_reset), 'stefcal_diffgain1.reset': int(reset or diffgain_reset), 'stefcal_ifr_gain_mode': "apply" if apply_only or ifrgain_apply_only else "solve-save", 'stefcal_ifr_gain_reset': int(reset or ifrgain_reset), 'stefcal_gain.table': STEFCAL_GAIN, 'stefcal_gain1.table': STEFCAL_GAIN1, 'stefcal_diffgain.table': STEFCAL_DIFFGAIN, 'stefcal_diffgain1.table': STEFCAL_DIFFGAIN1, 'stefcal_ifr_gain_table': STEFCAL_IFRGAIN, 'stefcal_visualize': False } # set gain parameters if gain_smoothing or STEFCAL_GAIN_SMOOTHING or gain_intervals or STEFCAL_GAIN_INTERVALS: timesmooth,freqsmooth = gain_smoothing or STEFCAL_GAIN_SMOOTHING or (0,0); timeint,freqint = gain_intervals or STEFCAL_GAIN_INTERVALS or (0,0); opts['stefcal_gain.timeint'] = 0 if timesmooth else timeint; opts['stefcal_gain.freqint'] = 0 if freqsmooth else freqint; opts['stefcal_gain.timesmooth'] = timesmooth; opts['stefcal_gain.freqsmooth'] = freqsmooth; # set diffgain parameters if diffgain_smoothing or STEFCAL_DIFFGAIN_SMOOTHING or diffgain_intervals or STEFCAL_DIFFGAIN_INTERVALS: timesmooth,freqsmooth = diffgain_smoothing or STEFCAL_DIFFGAIN_SMOOTHING or (0,0); timeint,freqint = diffgain_intervals or STEFCAL_DIFFGAIN_INTERVALS or (0,0); opts['stefcal_diffgain.timeint'] = 0 if timesmooth else timeint; opts['stefcal_diffgain.freqint'] = 0 if freqsmooth else freqint; opts['stefcal_diffgain.timesmooth'] = timesmooth; opts['stefcal_diffgain.freqsmooth'] = freqsmooth; # add user-defined args args0 += list(args); opts.update(options); opts.update(kws); # run the job mqt.run(STEFCAL_SCRIPT,STEFCAL_JOBNAME,section=section,saveconfig=saveconfig,args=args0,options=opts); # copy gains try: if not apply_only: if os.path.exists(STEFCAL_GAIN) and not gain_apply_only: std.copy(STEFCAL_GAIN,STEFCAL_GAIN_SAVE); if gain_plot_prefix: make_gain_plots(STEFCAL_GAIN_SAVE,prefix=gain_plot_prefix); if os.path.exists(STEFCAL_GAIN1) and not gain_apply_only: std.copy(STEFCAL_GAIN1,STEFCAL_GAIN1_SAVE); if gain_plot_prefix: make_gain_plots(STEFCAL_GAIN1_SAVE,prefix=gain_plot_prefix); if os.path.exists(STEFCAL_DIFFGAIN) and not diffgain_apply_only: std.copy(STEFCAL_DIFFGAIN,STEFCAL_DIFFGAIN_SAVE); if diffgain_plot_prefix: make_diffgain_plots(STEFCAL_DIFFGAIN_SAVE,prefix=diffgain_plot_prefix); if os.path.exists(STEFCAL_IFRGAIN) and not ifrgain_apply_only: std.copy(STEFCAL_IFRGAIN,STEFCAL_IFRGAIN_SAVE); if ifrgain_plot_prefix: make_ifrgain_plots(STEFCAL_IFRGAIN_SAVE,prefix=ifrgain_plot_prefix); except: traceback.print_exc(); plotfail("plot routine failed, see exception above"); # post-calibration flagging if flag_threshold: if isinstance(flag_threshold,(list,tuple)): t0,t1 = flag_threshold; else: t0,t1 = flag_threshold,None; ms.flagms("--above %g"%t0,"-f threshold -c"); if t1: ms.flagms("--fm-above %g"%t1,"-f fmthreshold -c"); # plot residuals if plotvis: try: info("Plotting visibilities ($plotvis)"); ms.PLOTVIS = plotvis; ms.plotms("-o ${OUTFILE}_${output}${_s<STEP}${_<label}.png"); except: traceback.print_exc(); plotfail("plot routine failed, see exception above"); # make images im.make_image(msname,column=STEFCAL_OUTPUT_COLUMN,dirty=dirty,restore=restore,restore_lsm=restore_lsm);