def simsky(msname="$MS", lsmname="$LSM", column="$COLUMN",
tdlconf="$TDLCONF", tdlsec="$SIMSEC", addnoise=True,
noise=0, sefd=0, recenter=True, options={} ,args=[],**kw):
"""
Simulates visibilities into a MS.
msname : MS name
lsmname : LSM name
column : Column to simulate visibilities into
tdlconf : Meqtrees TDL configuration profiles file (required to run MeqTrees pipeliner)
tdlsec : Section to execute in tdlconf
noise : Visibility noise to add to simulation.
args, kw : extra arguments to pass to the MeqTrees pipeliner
"""
msname,lsmname,column,tdlsec,tdlconf = interpolate_locals('msname lsmname column tdlsec tdlconf')
# recenter LSM if required
if recenter:
x.sh('tigger-convert --recenter=$DIRECTION $lsmname $RLSM -f')
v.LSM = RLSM
else:
v.LSM = lsmname
args = ["${ms.MS_TDL} ${lsm.LSM_TDL}"] + list(args)
options["pybeams_fits.filename_pattern"] = BEAM_PATTERN_HOL
options['ms_sel.output_column'] = column
if addnoise:
sefd = sefd or SEFD
options['noise_stddev'] = noise or compute_vis_noise(sefd)
options.update(kw) # extra keyword args get preference
mqt.run(TURBO_SIM,job='_tdl_job_1_simulate_MS',config=tdlconf,section=tdlsec,options=options,args=args)
def simsky(msname='$MS',
lsmname='$LSM',
tdlsec='$TDLSEC',
tdlconf='$TDLCONF',
column='$COLUMN',
noise=0,
args=[],
addToCol=None,
**kw):
""" Simulates visibilities into an MS """
msname, lsmname, column, tdlsec, tdlconf = interpolate_locals(
'msname lsmname'
' column tdlsec tdlconf')
fits = True if verify_sky(lsmname) is 'FITS' else False
v.MS = msname
v.LSM = lsmname
_column = 'MODEL_DATA' if addToCol else column
if fits:
_column = 'MODEL_DATA' if noise else column
im.argo.predict_vis(lsmname, wprojplanes=128, column=_column)
if noise:
simnoise(noise=noise, addToCol=_column, column=column)
else:
args = ["${ms.MS_TDL} ${lsm.LSM_TDL}"] + list(args)
options = {}
options['ms_sel.output_column'] = _column
if noise:
options['noise_stddev'] = noise
mqt.run(TURBO_SIM,
job='_tdl_job_1_simulate_MS',
config=tdlconf,
section=tdlsec,
options=options,
args=args,
**kw)
if addToCol:
tab = ms.msw()
col1 = tab.getcol(addToCol)
col2 = tab.getcol('MODEL_DATA')
comb = col1 + col2
nrows = len(comb)
rowchunk = nrows // 5
for row0 in range(0, nrows, rowchunk):
nr = min(nrows - row0, rowchunk)
info('MODEL_DATA + $addToCol --> $column : rows %d-%d' %
(row0, row0 + nr))
tab.putcol(column, comb[row0:row0 + nr], row0, nr)
tab.close()
def addnoise (noise=0,rowchunk=100000):
"""adds noise to MODEL_DATA, writes to CORRECTED_DATA""";
# compute expected noise
noise = compute_vis_noise(noise);
# fill MS with noise
# setup stefcal options and run
info("Running turbo-sim to add noise to data");
# setup args
args = [ """${ms.MS_TDL} ${ms.CHAN_TDL} ms_sel.ms_ifr_subset_str=${ms.IFRS} noise_stddev=%g"""%noise ];
mqt.run("${mqt.CATTERY}/Siamese/turbo-sim.py","simulate",section="addnoise",args=args);
def cal_function(type_cal="LM"):
options = {}
options['ms_sel.msname']=II("${MS}")
options['ms_sel.tile_size']=get_time_slots()
v.CALORNOT = "cal_model"
#options['ms_sel.output_column']="CORRECTED_DATA"
options['tiggerlsm.filename']=II("${LSM}")
if type_cal=="LM":
mqt.run(script="calico-wsrt-tens.py",job="cal_G_diag",config="tdlconf.profiles",section="selfcal",options=options)
elif type_cal=="STEF":
mqt.run(script="calico-stefcal.py",job="stefcal",config="tdlconf.profiles",section="G_calibration",options=options)
#elif type_cal=="STEF_DIR": #CALLING STEFCAL directly fails at the moment
# stefcal.stefcal(section="G_calibration",output="CORR_DATA",options=options)
v.CALORNOT = ''
def sim_function(cal=False,whole=False):
options = {}
options['ms_sel.msname']=II("${MS}")
options['ms_sel.tile_size']=get_time_slots()
if cal:
if not whole:
v.CALORNOT = "cal_model"
options['ms_sel.output_column']="DATA"
else:
options['ms_sel.output_column']="CORRECTED_DATA"
options['tiggerlsm.filename']=II("${LSM}")
mqt.run(script="turbo-sim.py",job="_tdl_job_1_simulate_MS",config="tdlconf.profiles",section="sim",options=options)
v.CALORNOT = ''
def addnoise(noise=0, rowchunk=100000):
"""adds noise to MODEL_DATA, writes to CORRECTED_DATA"""
# compute expected noise
noise = compute_vis_noise(noise)
# fill MS with noise
# setup stefcal options and run
info("Running turbo-sim to add noise to data")
# setup args
args = [
"""${ms.MS_TDL} ${ms.CHAN_TDL} ms_sel.ms_ifr_subset_str=${ms.IFRS} noise_stddev=%g"""
% noise
]
mqt.run("${mqt.CATTERY}/Siamese/turbo-sim.py",
"simulate",
section="addnoise",
args=args)
def simsky(msname="$MS",
lsmname="$LSM",
column="$COLUMN",
tdlconf="$TDLCONF",
tdlsec="$SIMSEC",
addnoise=True,
noise=0,
sefd=0,
recenter=True,
options={},
args=[],
**kw):
"""
Simulates visibilities into a MS.
msname : MS name
lsmname : LSM name
column : Column to simulate visibilities into
tdlconf : Meqtrees TDL configuration profiles file (required to run MeqTrees pipeliner)
tdlsec : Section to execute in tdlconf
noise : Visibility noise to add to simulation.
args, kw : extra arguments to pass to the MeqTrees pipeliner
"""
msname, lsmname, column, tdlsec, tdlconf = interpolate_locals(
'msname lsmname column tdlsec tdlconf')
# recenter LSM if required
if recenter:
x.sh('tigger-convert --recenter=$DIRECTION $lsmname $RLSM -f')
v.LSM = RLSM
else:
v.LSM = lsmname
args = ["${ms.MS_TDL} ${lsm.LSM_TDL}"] + list(args)
options["pybeams_fits.filename_pattern"] = BEAM_PATTERN_HOL
options['ms_sel.output_column'] = column
if addnoise:
sefd = sefd or SEFD
options['noise_stddev'] = noise or compute_vis_noise(sefd)
options.update(kw) # extra keyword args get preference
mqt.run(TURBO_SIM,
job='_tdl_job_1_simulate_MS',
config=tdlconf,
section=tdlsec,
options=options,
args=args)
def simsky(msname='$MS', lsmname='$LSM', tdlsec='$TDLSEC', tdlconf='$TDLCONF',
column='$COLUMN', noise=0, args=[],
addToCol=None,**kw):
""" Simulates visibilities into an MS """
msname, lsmname, column, tdlsec, tdlconf = interpolate_locals('msname lsmname'
' column tdlsec tdlconf')
fits = True if verify_sky(lsmname) is 'FITS' else False
v.MS = msname
v.LSM = lsmname
_column = 'MODEL_DATA' if addToCol else column
if fits:
_column = 'MODEL_DATA' if noise else column
im.argo.predict_vis(lsmname, wprojplanes=128, column=_column)
if noise:
simnoise(noise=noise,addToCol=_column,column=column)
else:
args = ["${ms.MS_TDL} ${lsm.LSM_TDL}"] + list(args)
options = {}
options['ms_sel.output_column'] = _column
if noise:
options['noise_stddev'] = noise
mqt.run(TURBO_SIM, job='_tdl_job_1_simulate_MS',
config=tdlconf, section=tdlsec, options=options, args=args,**kw)
if addToCol:
tab = ms.msw()
col1 = tab.getcol(addToCol)
col2 = tab.getcol('MODEL_DATA')
comb = col1 + col2
nrows = len(comb)
rowchunk = nrows//5
for row0 in range(0,nrows,rowchunk):
nr = min(nrows-row0,rowchunk)
info('MODEL_DATA + $addToCol --> $column : rows %d-%d'%(row0,row0+nr) )
tab.putcol(column,comb[row0:row0+nr],row0,nr)
tab.close()
def add_pjones(output_column):
options = {}
options['ms_sel.msname'] = II('$MS')
options['ms_sel.output_column'] = output_column
options['read_ms_model'] = 1 # read existing visibilities from MS
options['ms_sel.model_column'] = output_column
options['ms_sel.tile_size'] = 1000000
options['sim_mode'] = 'sim only'
options['me.p_enable'] = 1
options['feed_angle.enable_pa'] = 1 # enable parallactic angle rotation
options[
'feed_angle.read_ms'] = 1 # enable reading feed angle from FEED subtable in MS
mqt.MULTITHREAD = 32 #max number of meqserver threads
mqt.run(script=II('$FRAMEWORKDIR') + '/turbo-sim.py',
config=II('$FRAMEWORKDIR') + '/tdlconf.profiles',
job='_simulate_MS',
options=options)
def simulate(msname='$MS',
lsmname='$LSM',
tdlconf='$TDLCONF',
section='$TDLSEC',
freq0=1400e6,
options={},
**kw):
""" Simulates visibilities into an MS """
v.MS, v.LSM = interpolate_locals('msname lsmname')
if LSM:
options['tiggerlsm.filename'] = LSM
options['noise_stddev'] = NOISE or compute_vis_noise(sefd=get_sefd(freq0))
options['ms_sel.output_column'] = COLUMN
if USING_SIAMESE: section = 'turbo-sim:default'
options['ms_sel.select_channels'] = 0
mqt.run('turbo-sim.py',
job='_tdl_job_1_simulate_MS',
config=tdlconf,
section=section,
options=options)
def sim_now():
radius1 = np.arange(0., 100., 10.)
#radius1 = np.arange(0.,3,0.3)
Flux_array1 = np.zeros(len(radius1))
lores1 = "MeerKATlores.MS_p0_p0"
imager.cellsize = "10.arcsec"
imager.npix = 512
imager.niter = 0
imager.CLEAN_ALGORITHM = "csclean"
for k in range(len(radius1)):
options = {}
options['gridded_sky.grid_m0'] = radius1[k]
options['ms_sel.msname'] = lores1
mqt.run("turbo-sim.py",
job="_tdl_job_1_simulate_MS",
config="tdlconf.profiles1",
section="Sim_source_radius",
options=options)
center_min = -45 * 60 + radius1[k]
center_deg = math.ceil(center_min / 60)
center_min = abs(center_min - center_deg * 60)
imager.make_image(msname=lores1,
column='CORRECTED_DATA',
phasecenter="j2000,0h0m,%dd%dm" %
(center_deg, center_min),
restore=False,
dirty=True,
restore_lsm=False,
weight="natural")
f1 = np.max(pyfits.open(imager.DIRTY_IMAGE)[0].data[0][0])
Flux_array1[k] = f1
np.save("DATA/Flux1", Flux_array1)
np.save("DATA/radius1", radius1)
def run_turbosim(input_fitsimage, output_column, taql_string):
options = {}
options['ms_sel.msname'] = II('$MS')
options['ms_sel.output_column'] = output_column
if input_fitsimage.endswith(('.fits', '.FITS')):
options['me.sky.siamese_oms_fitsimage_sky'] = 1
options['fitsimage_sky.image_filename'] = input_fitsimage
options['fitsimage_sky.pad_factor'] = 2.4
elif input_fitsimage.endswith(('.txt', '.html')):
options['me.sky.tiggerskymodel'] = 1
options['tiggerlsm.filename'] = input_fitsimage
options['ms_sel.tile_size'] = 1000000
options['ms_sel.ms_taql_str'] = taql_string
mqt.MULTITHREAD = 32 #max number of meqserver threads
mqt.run(script=II('$FRAMEWORKDIR') + '/turbo-sim.py',
config=II('$FRAMEWORKDIR') + '/tdlconf.profiles',
section='turbo-sim',
job='_simulate_MS',
options=options)
def stefcal ( msname="$MS",section="$STEFCAL_SECTION",
diffgains=None,
apply_only=False,
gain_apply_only=False,
diffgain_apply_only=False,
ifrgain_apply_only=False,
diffgain_intervals=None,diffgain_smoothing=None,
flag_threshold=None,
output="CORR_RES",
plotvis="${ms.PLOTVIS}",
dirty=True,restore=False,restore_lsm=True,
label=None,
args=[],options={},
**kws):
"""Generic function to run a stefcal job.
'section' TDL config file section
'label' will be assigned to the global LABEL for purposes of file naming
'apply_only' if true, will only apply saved solutions
'diffgains' set to a source subset string to solve for diffgains. Set to True to use "=dE"
'diffgain_mode' 'solve-save' to solve & save, 'solve-nosave' to not save, 'apply' to apply only
'flag_threshold' threshold flaging post-solutions. Give one threshold to flag with --above,
or T1,T2 for --above T1 --fm-above T2
'output' output visibilities ('CORR_DATA','CORR_RES', 'RES' are useful)
'plotvis' if not empty, specifies which output visibilities to plot using plot-ms (see plot.ms.py --help)
'dirty','restore'
'restore_lsm' image output visibilities (passed to imager.make_image above as is)
'args','options' passed to the stefcal job as is (as a list of arguments and kw=value pairs),
can be used to supply extra TDL options
extra keywords: passed to the stefcal job as kw=value, can be used to supply extra TDL options, thus
overriding settings in the TDL config file. Useful arguments of this kind are e.g.:
stefcal_reset_all=True to remove prior gains solutions.
"""
msname,section,lsm,label,plotvis = interpolate_locals("msname section lsm label plotvis");
makedir(v.DESTDIR);
# increment step counter and assign global label
if label is not None:
v.LABEL = str(label);
if type(v.STEP) is int and STEFCAL_STEP_INCR:
v.STEP += STEFCAL_STEP_INCR;
# setup stefcal options and run
info("Running stefcal ${step <STEP} ${(<LABEL>)}");
# setup args
args0 = [ """${ms.MS_TDL} ${ms.CHAN_TDL} ${lsm.LSM_TDL} ms_sel.ms_ifr_subset_str=${ms.IFRS}
ms_sel.output_column=$STEFCAL_OUTPUT_COLUMN
stefcal_gain.enabled=1 stefcal_diffgain.enabled=%d %s"""%
((1 if diffgains else 0),STEFCAL_TDLOPTS) ];
if diffgains:
if diffgains is True:
diffgains = "=dE";
args0.append("de_subset.subset_enabled=1 de_subset.source_subset=$diffgains");
opts = {
'do_output': output,
'stefcal_gain.mode': "apply" if apply_only or gain_apply_only else "solve-save",
'stefcal_gain1.mode': "apply" if apply_only or gain_apply_only else "solve-save",
'stefcal_diffgain.mode': "apply" if apply_only or diffgain_apply_only else "solve-save",
'stefcal_diffgain1.mode': "apply" if apply_only or diffgain_apply_only else "solve-save",
'stefcal_ifr_gain_mode': "apply" if apply_only or ifrgain_apply_only else "solve-save",
'stefcal_gain.table': STEFCAL_GAIN,
'stefcal_gain1.table': STEFCAL_GAIN1,
'stefcal_diffgain.table': STEFCAL_DIFFGAIN,
'stefcal_diffgain1.table': STEFCAL_DIFFGAIN1,
'stefcal_ifr_gain_table': STEFCAL_IFRGAIN,
'stefcal_visualize': False
}
timesmooth,freqsmooth = diffgain_smoothing or STEFCAL_DIFFGAIN_SMOOTHING or (0,0);
timeint,freqint = diffgain_intervals or STEFCAL_DIFFGAIN_INTERVALS or (0,0);
opts['stefcal_diffgain.timeint'] = 0 if timesmooth else timeint;
opts['stefcal_diffgain.freqint'] = 0 if freqsmooth else freqint;
opts['stefcal_diffgain.timesmooth'] = timesmooth;
opts['stefcal_diffgain.freqsmooth'] = freqsmooth;
# add user-defined args
args0 += list(args);
opts.update(options);
opts.update(kws);
# run the job
mqt.run(STEFCAL_SCRIPT,STEFCAL_JOBNAME,section=section,args=args0,options=opts);
# copy gains
if not apply_only:
if os.path.exists(STEFCAL_GAIN):
std.copy(STEFCAL_GAIN,STEFCAL_GAIN_SAVE);
if os.path.exists(STEFCAL_GAIN1):
std.copy(STEFCAL_GAIN1,STEFCAL_GAIN1_SAVE);
if os.path.exists(STEFCAL_DIFFGAIN):
std.copy(STEFCAL_DIFFGAIN,STEFCAL_DIFFGAIN_SAVE);
if os.path.exists(STEFCAL_IFRGAIN):
std.copy(STEFCAL_IFRGAIN,STEFCAL_IFRGAIN_SAVE);
# post-calibration flagging
if flag_threshold:
if isinstance(flag_threshold,(list,tuple)):
t0,t1 = flag_threshold;
else:
t0,t1 = flag_threshold,None;
ms.flagms("--above %g"%t0,"-f threshold -c");
if t1:
ms.flagms("--fm-above %g"%t1,"-f fmthreshold -c");
# plot residuals
if plotvis:
info("Plotting visibilities ($plotvis)");
ms.PLOTVIS = plotvis;
ms.plotms("-o ${OUTFILE}_${output}${_s<STEP}${_<label}.png");
# make images
imager.make_image(msname,column=STEFCAL_OUTPUT_COLUMN,dirty=dirty,restore=restore,restore_lsm=restore_lsm);
def stefcal(msname="$MS",
section="$STEFCAL_SECTION",
diffgains=None,
apply_only=False,
reset=False,
gain_apply_only=False,
gain_reset=False,
diffgain_apply_only=False,
diffgain_reset=False,
gain_plot_prefix="$STEFCAL_GAIN_PLOT_PREFIX",
gain1_plot_prefix="$STEFCAL_GAIN1_PLOT_PREFIX",
ifrgain_plot_prefix="$STEFCAL_IFRGAIN_PLOT_PREFIX",
diffgain_plot_prefix="$STEFCAL_DIFFGAIN_PLOT_PREFIX",
ifrgain_apply_only=False,
ifrgain_reset=False,
gain_intervals=None,
gain_smoothing=None,
diffgain_intervals=None,
diffgain_smoothing=None,
flag_threshold=None,
calibrate_ifrs="$STEFCAL_CALIBRATE_IFRS",
input_column="$STEFCAL_INPUT_COLUMN",
output_column="$STEFCAL_OUTPUT_COLUMN",
output="CORR_RES",
plotvis="${ms.PLOTVIS}",
dirty=True,
restore=False,
restore_lsm=True,
label=None,
saveconfig="$STEFCAL_SAVE_CONFIG",
plotfail=None,
args=[],
options={},
**kws):
"""Generic function to run a stefcal job.
'section' TDL config file section
'label' will be assigned to the global LABEL for purposes of file naming
'apply_only' if true, will only apply saved solutions rather than re-solve
'{gain,diffgain,ifrgain}_apply_only'
if true, will only apply saved gain/diffgain/IFR gain solutions rather than re-solve
'reset' if true, will reset all saved solutuions prior to starting
'{gain,diffgain,ifrgain}_reset'
if true, will reset saved gain/diffgain/IFR gain solutuions prior to starting
'diffgains' set to a source subset string to solve for diffgains. Set to True to use "=dE"
'diffgain_mode' 'solve-save' to solve & save, 'solve-nosave' to not save, 'apply' to apply only
'diffgain_plot' automatically invoke make_diffgain_plots() if True
'flag_threshold' threshold flaging post-solutions. Give one threshold to flag with --above,
or T1,T2 for --above T1 --fm-above T2
'output' output visibilities ('CORR_DATA','CORR_RES', 'RES' are useful)
'plotvis' if not empty, specifies which output visibilities to plot using plot-ms (see plot.ms.py --help)
'dirty','restore'
'restore_lsm' image output visibilities (passed to imager.make_image above as is)
'plotfail' plotting failure reported via warn or abort. Default is warn, set to abort to abort.
'saveconfig' saves the effective TDL config to file[:section]
'args','options' passed to the stefcal job as is (as a list of arguments and kw=value pairs),
can be used to supply extra TDL options
extra keywords: passed to the stefcal job as kw=value, can be used to supply extra TDL options, thus
overriding settings in the TDL config file. Useful arguments of this kind are e.g.:
stefcal_reset_all=True to remove prior gains solutions.
"""
msname,section,lsm,label,plotvis,calibrate_ifrs, \
gain_plot_prefix,gain1_plot_prefix,ifrgain_plot_prefix,diffgain_plot_prefix,saveconfig,plotfail,input_column,output_column = \
interpolate_locals("msname section lsm label plotvis calibrate_ifrs gain_plot_prefix gain1_plot_prefix ifrgain_plot_prefix diffgain_plot_prefix "
"saveconfig plotfail input_column output_column")
plotfail = plotfail or warn
makedir(v.DESTDIR)
# increment step counter and assign global label
if label is not None:
v.LABEL = str(label)
if type(v.STEP) is int and STEFCAL_STEP_INCR:
v.STEP += STEFCAL_STEP_INCR
# setup stefcal options and run
info("Running stefcal ${step <STEP} ${(<LABEL>)}")
# setup args
args0 = [
"""${ms.MS_TDL} ${ms.CHAN_TDL} ${lsm.LSM_TDL} ms_sel.ms_ifr_subset_str=${ms.IFRS}
ms_sel.input_column=$input_column
ms_sel.output_column=$output_column
stefcal_gain.enabled=1 stefcal_diffgain.enabled=%d %s""" %
((1 if diffgains else 0), STEFCAL_TDLOPTS)
]
if diffgains:
if diffgains is True:
diffgains = "=dE"
args0.append(
"de_subset.subset_enabled=1 de_subset.source_subset=$diffgains")
opts = {
'do_output':
output,
'calibrate_ifrs':
calibrate_ifrs,
'stefcal_gain.mode':
"apply" if apply_only or gain_apply_only else "solve-save",
'stefcal_gain1.mode':
"apply" if apply_only or gain_apply_only else "solve-save",
'stefcal_gain.reset':
int(reset or gain_reset),
'stefcal_diffgain.mode':
"apply" if apply_only or diffgain_apply_only else "solve-save",
'stefcal_diffgain1.mode':
"apply" if apply_only or diffgain_apply_only else "solve-save",
'stefcal_diffgain.reset':
int(reset or diffgain_reset),
'stefcal_diffgain1.reset':
int(reset or diffgain_reset),
'stefcal_ifr_gain_mode':
"apply" if apply_only or ifrgain_apply_only else "solve-save",
'stefcal_ifr_gain_reset':
int(reset or ifrgain_reset),
'stefcal_gain.table':
STEFCAL_GAIN,
'stefcal_gain1.table':
STEFCAL_GAIN1,
'stefcal_diffgain.table':
STEFCAL_DIFFGAIN,
'stefcal_diffgain1.table':
STEFCAL_DIFFGAIN1,
'stefcal_ifr_gain_table':
STEFCAL_IFRGAIN,
'stefcal_visualize':
False
}
# set gain parameters
if gain_smoothing or STEFCAL_GAIN_SMOOTHING or gain_intervals or STEFCAL_GAIN_INTERVALS:
timesmooth, freqsmooth = gain_smoothing or STEFCAL_GAIN_SMOOTHING or (
0, 0)
timeint, freqint = gain_intervals or STEFCAL_GAIN_INTERVALS or (0, 0)
opts['stefcal_gain.timeint'] = 0 if timesmooth else timeint
opts['stefcal_gain.freqint'] = 0 if freqsmooth else freqint
opts['stefcal_gain.timesmooth'] = timesmooth
opts['stefcal_gain.freqsmooth'] = freqsmooth
# set diffgain parameters
if diffgain_smoothing or STEFCAL_DIFFGAIN_SMOOTHING or diffgain_intervals or STEFCAL_DIFFGAIN_INTERVALS:
timesmooth, freqsmooth = diffgain_smoothing or STEFCAL_DIFFGAIN_SMOOTHING or (
0, 0)
timeint, freqint = diffgain_intervals or STEFCAL_DIFFGAIN_INTERVALS or (
0, 0)
opts['stefcal_diffgain.timeint'] = 0 if timesmooth else timeint
opts['stefcal_diffgain.freqint'] = 0 if freqsmooth else freqint
opts['stefcal_diffgain.timesmooth'] = timesmooth
opts['stefcal_diffgain.freqsmooth'] = freqsmooth
# add user-defined args
args0 += list(args)
opts.update(options)
opts.update(kws)
# run the job
mqt.run(STEFCAL_SCRIPT,
STEFCAL_JOBNAME,
section=section,
saveconfig=saveconfig,
args=args0,
options=opts)
# copy gains
try:
if not apply_only:
if os.path.exists(STEFCAL_GAIN) and not gain_apply_only:
std.copy(STEFCAL_GAIN, STEFCAL_GAIN_SAVE)
if gain_plot_prefix:
make_gain_plots(STEFCAL_GAIN_SAVE, prefix=gain_plot_prefix)
if os.path.exists(STEFCAL_GAIN1) and not gain_apply_only:
std.copy(STEFCAL_GAIN1, STEFCAL_GAIN1_SAVE)
if gain_plot_prefix:
make_gain_plots(STEFCAL_GAIN1_SAVE,
prefix=gain_plot_prefix)
if os.path.exists(STEFCAL_DIFFGAIN) and not diffgain_apply_only:
std.copy(STEFCAL_DIFFGAIN, STEFCAL_DIFFGAIN_SAVE)
if diffgain_plot_prefix:
make_diffgain_plots(STEFCAL_DIFFGAIN_SAVE,
prefix=diffgain_plot_prefix)
if os.path.exists(STEFCAL_IFRGAIN) and not ifrgain_apply_only:
std.copy(STEFCAL_IFRGAIN, STEFCAL_IFRGAIN_SAVE)
if ifrgain_plot_prefix:
make_ifrgain_plots(STEFCAL_IFRGAIN_SAVE,
prefix=ifrgain_plot_prefix)
except:
traceback.print_exc()
plotfail("plot routine failed, see exception above")
# post-calibration flagging
if flag_threshold:
if isinstance(flag_threshold, (list, tuple)):
t0, t1 = flag_threshold
else:
t0, t1 = flag_threshold, None
ms.flagms("--above %g" % t0, "-f threshold -c")
if t1:
ms.flagms("--fm-above %g" % t1, "-f fmthreshold -c")
# plot residuals
if plotvis:
try:
info("Plotting visibilities ($plotvis)")
ms.PLOTVIS = plotvis
ms.plotms("-o ${OUTFILE}_${output}${_s<STEP}${_<label}.png")
except:
traceback.print_exc()
plotfail("plot routine failed, see exception above")
# make images
im.make_image(msname,
column=STEFCAL_OUTPUT_COLUMN,
dirty=dirty,
restore=restore,
restore_lsm=restore_lsm)
def stefcal(msname="$MS",
section="$STEFCAL_SECTION",
diffgains=None,
apply_only=False,
gain_apply_only=False,
diffgain_apply_only=False,
ifrgain_apply_only=False,
diffgain_intervals=None,
diffgain_smoothing=None,
flag_threshold=None,
output="CORR_RES",
plotvis="${ms.PLOTVIS}",
dirty=True,
restore=False,
restore_lsm=True,
label=None,
args=[],
options={},
**kws):
"""Generic function to run a stefcal job.
'section' TDL config file section
'label' will be assigned to the global LABEL for purposes of file naming
'apply_only' if true, will only apply saved solutions
'diffgains' set to a source subset string to solve for diffgains. Set to True to use "=dE"
'diffgain_mode' 'solve-save' to solve & save, 'solve-nosave' to not save, 'apply' to apply only
'flag_threshold' threshold flaging post-solutions. Give one threshold to flag with --above,
or T1,T2 for --above T1 --fm-above T2
'output' output visibilities ('CORR_DATA','CORR_RES', 'RES' are useful)
'plotvis' if not empty, specifies which output visibilities to plot using plot-ms (see plot.ms.py --help)
'dirty','restore'
'restore_lsm' image output visibilities (passed to imager.make_image above as is)
'args','options' passed to the stefcal job as is (as a list of arguments and kw=value pairs),
can be used to supply extra TDL options
extra keywords: passed to the stefcal job as kw=value, can be used to supply extra TDL options, thus
overriding settings in the TDL config file. Useful arguments of this kind are e.g.:
stefcal_reset_all=True to remove prior gains solutions.
"""
msname, section, lsm, label, plotvis = interpolate_locals(
"msname section lsm label plotvis")
makedir(v.DESTDIR)
# increment step counter and assign global label
if label is not None:
v.LABEL = str(label)
if type(v.STEP) is int and STEFCAL_STEP_INCR:
v.STEP += STEFCAL_STEP_INCR
# setup stefcal options and run
info("Running stefcal ${step <STEP} ${(<LABEL>)}")
# setup args
args0 = [
"""${ms.MS_TDL} ${ms.CHAN_TDL} ${lsm.LSM_TDL} ms_sel.ms_ifr_subset_str=${ms.IFRS}
ms_sel.output_column=$STEFCAL_OUTPUT_COLUMN
stefcal_gain.enabled=1 stefcal_diffgain.enabled=%d %s""" %
((1 if diffgains else 0), STEFCAL_TDLOPTS)
]
if diffgains:
if diffgains is True:
diffgains = "=dE"
args0.append(
"de_subset.subset_enabled=1 de_subset.source_subset=$diffgains")
opts = {
'do_output':
output,
'stefcal_gain.mode':
"apply" if apply_only or gain_apply_only else "solve-save",
'stefcal_gain1.mode':
"apply" if apply_only or gain_apply_only else "solve-save",
'stefcal_diffgain.mode':
"apply" if apply_only or diffgain_apply_only else "solve-save",
'stefcal_diffgain1.mode':
"apply" if apply_only or diffgain_apply_only else "solve-save",
'stefcal_ifr_gain_mode':
"apply" if apply_only or ifrgain_apply_only else "solve-save",
'stefcal_gain.table':
STEFCAL_GAIN,
'stefcal_gain1.table':
STEFCAL_GAIN1,
'stefcal_diffgain.table':
STEFCAL_DIFFGAIN,
'stefcal_diffgain1.table':
STEFCAL_DIFFGAIN1,
'stefcal_ifr_gain_table':
STEFCAL_IFRGAIN,
'stefcal_visualize':
False
}
timesmooth, freqsmooth = diffgain_smoothing or STEFCAL_DIFFGAIN_SMOOTHING or (
0, 0)
timeint, freqint = diffgain_intervals or STEFCAL_DIFFGAIN_INTERVALS or (0,
0)
opts['stefcal_diffgain.timeint'] = 0 if timesmooth else timeint
opts['stefcal_diffgain.freqint'] = 0 if freqsmooth else freqint
opts['stefcal_diffgain.timesmooth'] = timesmooth
opts['stefcal_diffgain.freqsmooth'] = freqsmooth
# add user-defined args
args0 += list(args)
opts.update(options)
opts.update(kws)
# run the job
mqt.run(STEFCAL_SCRIPT,
STEFCAL_JOBNAME,
section=section,
args=args0,
options=opts)
# copy gains
if not apply_only:
if os.path.exists(STEFCAL_GAIN):
std.copy(STEFCAL_GAIN, STEFCAL_GAIN_SAVE)
if os.path.exists(STEFCAL_GAIN1):
std.copy(STEFCAL_GAIN1, STEFCAL_GAIN1_SAVE)
if os.path.exists(STEFCAL_DIFFGAIN):
std.copy(STEFCAL_DIFFGAIN, STEFCAL_DIFFGAIN_SAVE)
if os.path.exists(STEFCAL_IFRGAIN):
std.copy(STEFCAL_IFRGAIN, STEFCAL_IFRGAIN_SAVE)
# post-calibration flagging
if flag_threshold:
if isinstance(flag_threshold, (list, tuple)):
t0, t1 = flag_threshold
else:
t0, t1 = flag_threshold, None
ms.flagms("--above %g" % t0, "-f threshold -c")
if t1:
ms.flagms("--fm-above %g" % t1, "-f fmthreshold -c")
# plot residuals
if plotvis:
info("Plotting visibilities ($plotvis)")
ms.PLOTVIS = plotvis
ms.plotms("-o ${OUTFILE}_${output}${_s<STEP}${_<label}.png")
# make images
imager.make_image(msname,
column=STEFCAL_OUTPUT_COLUMN,
dirty=dirty,
restore=restore,
restore_lsm=restore_lsm)
def stefcal ( msname="$MS",section="$STEFCAL_SECTION",
diffgains=None,
apply_only=False,
reset=False,
gain_apply_only=False,
gain_reset=False,
diffgain_apply_only=False,
diffgain_reset=False,
gain_plot_prefix="$STEFCAL_GAIN_PLOT_PREFIX",
gain1_plot_prefix="$STEFCAL_GAIN1_PLOT_PREFIX",
ifrgain_plot_prefix="$STEFCAL_IFRGAIN_PLOT_PREFIX",
diffgain_plot_prefix="$STEFCAL_DIFFGAIN_PLOT_PREFIX",
ifrgain_apply_only=False,
ifrgain_reset=False,
gain_intervals=None,gain_smoothing=None,
diffgain_intervals=None,diffgain_smoothing=None,
flag_threshold=None,
calibrate_ifrs="$STEFCAL_CALIBRATE_IFRS",
input_column="$STEFCAL_INPUT_COLUMN",
output_column="$STEFCAL_OUTPUT_COLUMN",
output="CORR_RES",
plotvis="${ms.PLOTVIS}",
dirty=True,restore=False,restore_lsm=True,
label=None,
saveconfig="$STEFCAL_SAVE_CONFIG",
plotfail=None,
args=[],options={},
**kws):
"""Generic function to run a stefcal job.
'section' TDL config file section
'label' will be assigned to the global LABEL for purposes of file naming
'apply_only' if true, will only apply saved solutions rather than re-solve
'{gain,diffgain,ifrgain}_apply_only'
if true, will only apply saved gain/diffgain/IFR gain solutions rather than re-solve
'reset' if true, will reset all saved solutuions prior to starting
'{gain,diffgain,ifrgain}_reset'
if true, will reset saved gain/diffgain/IFR gain solutuions prior to starting
'diffgains' set to a source subset string to solve for diffgains. Set to True to use "=dE"
'diffgain_mode' 'solve-save' to solve & save, 'solve-nosave' to not save, 'apply' to apply only
'diffgain_plot' automatically invoke make_diffgain_plots() if True
'flag_threshold' threshold flaging post-solutions. Give one threshold to flag with --above,
or T1,T2 for --above T1 --fm-above T2
'output' output visibilities ('CORR_DATA','CORR_RES', 'RES' are useful)
'plotvis' if not empty, specifies which output visibilities to plot using plot-ms (see plot.ms.py --help)
'dirty','restore'
'restore_lsm' image output visibilities (passed to imager.make_image above as is)
'plotfail' plotting failure reported via warn or abort. Default is warn, set to abort to abort.
'saveconfig' saves the effective TDL config to file[:section]
'args','options' passed to the stefcal job as is (as a list of arguments and kw=value pairs),
can be used to supply extra TDL options
extra keywords: passed to the stefcal job as kw=value, can be used to supply extra TDL options, thus
overriding settings in the TDL config file. Useful arguments of this kind are e.g.:
stefcal_reset_all=True to remove prior gains solutions.
"""
msname,section,lsm,label,plotvis,calibrate_ifrs, \
gain_plot_prefix,gain1_plot_prefix,ifrgain_plot_prefix,diffgain_plot_prefix,saveconfig,plotfail,input_column,output_column = \
interpolate_locals("msname section lsm label plotvis calibrate_ifrs gain_plot_prefix gain1_plot_prefix ifrgain_plot_prefix diffgain_plot_prefix "
"saveconfig plotfail input_column output_column")
plotfail = plotfail or warn
makedir(v.DESTDIR);
# increment step counter and assign global label
if label is not None:
v.LABEL = str(label);
if type(v.STEP) is int and STEFCAL_STEP_INCR:
v.STEP += STEFCAL_STEP_INCR;
# setup stefcal options and run
info("Running stefcal ${step <STEP} ${(<LABEL>)}");
# setup args
args0 = [ """${ms.MS_TDL} ${ms.CHAN_TDL} ${lsm.LSM_TDL} ms_sel.ms_ifr_subset_str=${ms.IFRS}
ms_sel.input_column=$input_column
ms_sel.output_column=$output_column
stefcal_gain.enabled=1 stefcal_diffgain.enabled=%d %s"""%
((1 if diffgains else 0),STEFCAL_TDLOPTS) ];
if diffgains:
if diffgains is True:
diffgains = "=dE";
args0.append("de_subset.subset_enabled=1 de_subset.source_subset=$diffgains");
opts = {
'do_output': output,
'calibrate_ifrs': calibrate_ifrs,
'stefcal_gain.mode': "apply" if apply_only or gain_apply_only else "solve-save",
'stefcal_gain1.mode': "apply" if apply_only or gain_apply_only else "solve-save",
'stefcal_gain.reset': int(reset or gain_reset),
'stefcal_diffgain.mode': "apply" if apply_only or diffgain_apply_only else "solve-save",
'stefcal_diffgain1.mode': "apply" if apply_only or diffgain_apply_only else "solve-save",
'stefcal_diffgain.reset': int(reset or diffgain_reset),
'stefcal_diffgain1.reset': int(reset or diffgain_reset),
'stefcal_ifr_gain_mode': "apply" if apply_only or ifrgain_apply_only else "solve-save",
'stefcal_ifr_gain_reset': int(reset or ifrgain_reset),
'stefcal_gain.table': STEFCAL_GAIN,
'stefcal_gain1.table': STEFCAL_GAIN1,
'stefcal_diffgain.table': STEFCAL_DIFFGAIN,
'stefcal_diffgain1.table': STEFCAL_DIFFGAIN1,
'stefcal_ifr_gain_table': STEFCAL_IFRGAIN,
'stefcal_visualize': False
}
# set gain parameters
if gain_smoothing or STEFCAL_GAIN_SMOOTHING or gain_intervals or STEFCAL_GAIN_INTERVALS:
timesmooth,freqsmooth = gain_smoothing or STEFCAL_GAIN_SMOOTHING or (0,0);
timeint,freqint = gain_intervals or STEFCAL_GAIN_INTERVALS or (0,0);
opts['stefcal_gain.timeint'] = 0 if timesmooth else timeint;
opts['stefcal_gain.freqint'] = 0 if freqsmooth else freqint;
opts['stefcal_gain.timesmooth'] = timesmooth;
opts['stefcal_gain.freqsmooth'] = freqsmooth;
# set diffgain parameters
if diffgain_smoothing or STEFCAL_DIFFGAIN_SMOOTHING or diffgain_intervals or STEFCAL_DIFFGAIN_INTERVALS:
timesmooth,freqsmooth = diffgain_smoothing or STEFCAL_DIFFGAIN_SMOOTHING or (0,0);
timeint,freqint = diffgain_intervals or STEFCAL_DIFFGAIN_INTERVALS or (0,0);
opts['stefcal_diffgain.timeint'] = 0 if timesmooth else timeint;
opts['stefcal_diffgain.freqint'] = 0 if freqsmooth else freqint;
opts['stefcal_diffgain.timesmooth'] = timesmooth;
opts['stefcal_diffgain.freqsmooth'] = freqsmooth;
# add user-defined args
args0 += list(args);
opts.update(options);
opts.update(kws);
# run the job
mqt.run(STEFCAL_SCRIPT,STEFCAL_JOBNAME,section=section,saveconfig=saveconfig,args=args0,options=opts);
# copy gains
try:
if not apply_only:
if os.path.exists(STEFCAL_GAIN) and not gain_apply_only:
std.copy(STEFCAL_GAIN,STEFCAL_GAIN_SAVE);
if gain_plot_prefix:
make_gain_plots(STEFCAL_GAIN_SAVE,prefix=gain_plot_prefix);
if os.path.exists(STEFCAL_GAIN1) and not gain_apply_only:
std.copy(STEFCAL_GAIN1,STEFCAL_GAIN1_SAVE);
if gain_plot_prefix:
make_gain_plots(STEFCAL_GAIN1_SAVE,prefix=gain_plot_prefix);
if os.path.exists(STEFCAL_DIFFGAIN) and not diffgain_apply_only:
std.copy(STEFCAL_DIFFGAIN,STEFCAL_DIFFGAIN_SAVE);
if diffgain_plot_prefix:
make_diffgain_plots(STEFCAL_DIFFGAIN_SAVE,prefix=diffgain_plot_prefix);
if os.path.exists(STEFCAL_IFRGAIN) and not ifrgain_apply_only:
std.copy(STEFCAL_IFRGAIN,STEFCAL_IFRGAIN_SAVE);
if ifrgain_plot_prefix:
make_ifrgain_plots(STEFCAL_IFRGAIN_SAVE,prefix=ifrgain_plot_prefix);
except:
traceback.print_exc();
plotfail("plot routine failed, see exception above");
# post-calibration flagging
if flag_threshold:
if isinstance(flag_threshold,(list,tuple)):
t0,t1 = flag_threshold;
else:
t0,t1 = flag_threshold,None;
ms.flagms("--above %g"%t0,"-f threshold -c");
if t1:
ms.flagms("--fm-above %g"%t1,"-f fmthreshold -c");
# plot residuals
if plotvis:
try:
info("Plotting visibilities ($plotvis)");
ms.PLOTVIS = plotvis;
ms.plotms("-o ${OUTFILE}_${output}${_s<STEP}${_<label}.png");
except:
traceback.print_exc();
plotfail("plot routine failed, see exception above");
# make images
im.make_image(msname,column=STEFCAL_OUTPUT_COLUMN,dirty=dirty,restore=restore,restore_lsm=restore_lsm);
