def reset_ms():
"""reset_ms: make a clean start.
If FULL_RESET is set (or MS does not exist): does a complete refresh. Removes the measurement set given by the MS variable,
untars a fresh copy from the data dir, and does various WSRT-specific initialization.
If FULL_RESET is not set, then simply clears out flagsets raised during calibration."""
if is_true("$FULL_RESET") or not exists(MS):
if exists(MS):
info(
"$MS exists but FULL_RESET=1, removing and unpacking fresh MS from tarball"
)
x.sh("rm -fr $MS")
else:
info("$MS does not exist, unpacking fresh MS from tarball")
x.sh("cd ${MS:DIR}; tar zxvf ${MS_TARBALL_DIR}/${MS:FILE}.tgz")
# init MODEL_DATA/CORRECTED_DATA/etc. columns
pyrap.tables.addImagingColumns(MS)
# add a bitflag column (used by MeqTrees)
ms.addbitflagcol()
# convert UVWs to J2000
ms.wsrt_j2convert()
# change the WEIGHT column of redundant baselines, this reduces grating lobes
# in images
ms.downweigh_redundant()
# use the flag-ms.py tool to copy the FLAG column to the "legacy" bitflag
ms.flagms("-Y +L -f legacy -c")
else:
info(
"$MS exists, reusing (but clearing calibration-related flags). Use FULL_RESET=1 to unpack fresh MS from tarball"
)
ms.flagms("-r threshold,fmthreshold,aoflagger,stefcal")
def reset_ms ():
"""reset_ms: make a clean start.
If FULL_RESET is set (or MS does not exist): does a complete refresh. Removes the measurement set given by the MS variable,
untars a fresh copy from the data dir, and does various WSRT-specific initialization.
If FULL_RESET is not set, then simply clears out flagsets raised during calibration."""
if is_true("$FULL_RESET") or not exists(MS):
if exists(MS):
info("$MS exists but FULL_RESET=1, removing and unpacking fresh MS from tarball");
x.sh("rm -fr $MS")
else:
info("$MS does not exist, unpacking fresh MS from tarball");
x.sh("cd ${MS:DIR}; tar zxvf ${MS_TARBALL_DIR}/${MS:FILE}.tgz")
# init MODEL_DATA/CORRECTED_DATA/etc. columns
pyrap.tables.addImagingColumns(MS)
# add a bitflag column (used by MeqTrees)
ms.addbitflagcol();
# convert UVWs to J2000
ms.wsrt_j2convert()
# change the WEIGHT column of redundant baselines, this reduces grating lobes
# in images
ms.downweigh_redundant()
# use the flag-ms.py tool to copy the FLAG column to the "legacy" bitflag
ms.flagms("-Y +L -f legacy -c")
else:
info("$MS exists, reusing (but clearing calibration-related flags). Use FULL_RESET=1 to unpack fresh MS from tarball");
ms.flagms("-r threshold,fmthreshold,aoflagger,stefcal");
def reset_ms ():
"""reset_ms: make a clean start.
If FULL_RESET is set (or MS does not exist): does a complete refresh. Removes the measurement set given by the MS variable,
If FULL_RESET is not set, then simply clears out flagsets raised during calibration."""
if is_true("$FULL_RESET"):
pyrap.tables.addImagingColumns(MS)
# add a bitflag column (used by MeqTrees)
ms.addbitflagcol();
# use the flag-ms.py tool to copy the FLAG column to the "legacy" bitflag
ms.flagms("-Y +L -f legacy -c")
else:
info("$MS exists, reusing (but clearing calibration-related flags). Use FULL_RESET=1 to unpack fresh MS from tarball");
ms.flagms("-r threshold,fmthreshold,aoflagger,stefcal");
def reset_ms():
"""reset_ms: make a clean start.
If FULL_RESET is set (or MS does not exist): does a complete refresh. Removes the measurement set given by the MS variable,
If FULL_RESET is not set, then simply clears out flagsets raised during calibration."""
if is_true("$FULL_RESET"):
pyrap.tables.addImagingColumns(MS)
# add a bitflag column (used by MeqTrees)
ms.addbitflagcol()
# use the flag-ms.py tool to copy the FLAG column to the "legacy" bitflag
ms.flagms("-Y +L -f legacy -c")
else:
info(
"$MS exists, reusing (but clearing calibration-related flags). Use FULL_RESET=1 to unpack fresh MS from tarball"
)
ms.flagms("-r threshold,fmthreshold,aoflagger,stefcal")
def reset_ms():
"""reset_ms: make a clean start.
If FULL_RESET is set (or MS does not exist): does a complete refresh. Removes the measurement set given by the MS variable,
untars a fresh copy from the data dir, and does various WSRT-specific initialization.
If FULL_RESET is not set, then simply clears out flagsets raised during calibration."""
# init MODEL_DATA/CORRECTED_DATA/etc. columns
#pyrap.tables.addImagingColumns(MS)
print "resetting"
# add a bitflag column (used by MeqTrees)
#cal.addbitflagcol(MS)
ms.addbitflagcol(MS)
# convert UVWs to J2000
# change the WEIGHT column of redundant baselines, this reduces grating lobes
# in images
# use the flag-ms.py tool to copy the FLAG column to the "legacy" bitflag
ms.flagms(MS, "-Y +L -f legacy -c")
def stefcal ( msname="$MS",section="$STEFCAL_SECTION",
diffgains=None,
apply_only=False,
gain_apply_only=False,
diffgain_apply_only=False,
ifrgain_apply_only=False,
diffgain_intervals=None,diffgain_smoothing=None,
flag_threshold=None,
output="CORR_RES",
plotvis="${ms.PLOTVIS}",
dirty=True,restore=False,restore_lsm=True,
label=None,
args=[],options={},
**kws):
"""Generic function to run a stefcal job.
'section' TDL config file section
'label' will be assigned to the global LABEL for purposes of file naming
'apply_only' if true, will only apply saved solutions
'diffgains' set to a source subset string to solve for diffgains. Set to True to use "=dE"
'diffgain_mode' 'solve-save' to solve & save, 'solve-nosave' to not save, 'apply' to apply only
'flag_threshold' threshold flaging post-solutions. Give one threshold to flag with --above,
or T1,T2 for --above T1 --fm-above T2
'output' output visibilities ('CORR_DATA','CORR_RES', 'RES' are useful)
'plotvis' if not empty, specifies which output visibilities to plot using plot-ms (see plot.ms.py --help)
'dirty','restore'
'restore_lsm' image output visibilities (passed to imager.make_image above as is)
'args','options' passed to the stefcal job as is (as a list of arguments and kw=value pairs),
can be used to supply extra TDL options
extra keywords: passed to the stefcal job as kw=value, can be used to supply extra TDL options, thus
overriding settings in the TDL config file. Useful arguments of this kind are e.g.:
stefcal_reset_all=True to remove prior gains solutions.
"""
msname,section,lsm,label,plotvis = interpolate_locals("msname section lsm label plotvis");
makedir(v.DESTDIR);
# increment step counter and assign global label
if label is not None:
v.LABEL = str(label);
if type(v.STEP) is int and STEFCAL_STEP_INCR:
v.STEP += STEFCAL_STEP_INCR;
# setup stefcal options and run
info("Running stefcal ${step <STEP} ${(<LABEL>)}");
# setup args
args0 = [ """${ms.MS_TDL} ${ms.CHAN_TDL} ${lsm.LSM_TDL} ms_sel.ms_ifr_subset_str=${ms.IFRS}
ms_sel.output_column=$STEFCAL_OUTPUT_COLUMN
stefcal_gain.enabled=1 stefcal_diffgain.enabled=%d %s"""%
((1 if diffgains else 0),STEFCAL_TDLOPTS) ];
if diffgains:
if diffgains is True:
diffgains = "=dE";
args0.append("de_subset.subset_enabled=1 de_subset.source_subset=$diffgains");
opts = {
'do_output': output,
'stefcal_gain.mode': "apply" if apply_only or gain_apply_only else "solve-save",
'stefcal_gain1.mode': "apply" if apply_only or gain_apply_only else "solve-save",
'stefcal_diffgain.mode': "apply" if apply_only or diffgain_apply_only else "solve-save",
'stefcal_diffgain1.mode': "apply" if apply_only or diffgain_apply_only else "solve-save",
'stefcal_ifr_gain_mode': "apply" if apply_only or ifrgain_apply_only else "solve-save",
'stefcal_gain.table': STEFCAL_GAIN,
'stefcal_gain1.table': STEFCAL_GAIN1,
'stefcal_diffgain.table': STEFCAL_DIFFGAIN,
'stefcal_diffgain1.table': STEFCAL_DIFFGAIN1,
'stefcal_ifr_gain_table': STEFCAL_IFRGAIN,
'stefcal_visualize': False
}
timesmooth,freqsmooth = diffgain_smoothing or STEFCAL_DIFFGAIN_SMOOTHING or (0,0);
timeint,freqint = diffgain_intervals or STEFCAL_DIFFGAIN_INTERVALS or (0,0);
opts['stefcal_diffgain.timeint'] = 0 if timesmooth else timeint;
opts['stefcal_diffgain.freqint'] = 0 if freqsmooth else freqint;
opts['stefcal_diffgain.timesmooth'] = timesmooth;
opts['stefcal_diffgain.freqsmooth'] = freqsmooth;
# add user-defined args
args0 += list(args);
opts.update(options);
opts.update(kws);
# run the job
mqt.run(STEFCAL_SCRIPT,STEFCAL_JOBNAME,section=section,args=args0,options=opts);
# copy gains
if not apply_only:
if os.path.exists(STEFCAL_GAIN):
std.copy(STEFCAL_GAIN,STEFCAL_GAIN_SAVE);
if os.path.exists(STEFCAL_GAIN1):
std.copy(STEFCAL_GAIN1,STEFCAL_GAIN1_SAVE);
if os.path.exists(STEFCAL_DIFFGAIN):
std.copy(STEFCAL_DIFFGAIN,STEFCAL_DIFFGAIN_SAVE);
if os.path.exists(STEFCAL_IFRGAIN):
std.copy(STEFCAL_IFRGAIN,STEFCAL_IFRGAIN_SAVE);
# post-calibration flagging
if flag_threshold:
if isinstance(flag_threshold,(list,tuple)):
t0,t1 = flag_threshold;
else:
t0,t1 = flag_threshold,None;
ms.flagms("--above %g"%t0,"-f threshold -c");
if t1:
ms.flagms("--fm-above %g"%t1,"-f fmthreshold -c");
# plot residuals
if plotvis:
info("Plotting visibilities ($plotvis)");
ms.PLOTVIS = plotvis;
ms.plotms("-o ${OUTFILE}_${output}${_s<STEP}${_<label}.png");
# make images
imager.make_image(msname,column=STEFCAL_OUTPUT_COLUMN,dirty=dirty,restore=restore,restore_lsm=restore_lsm);
def stefcal(msname="$MS",
section="$STEFCAL_SECTION",
diffgains=None,
apply_only=False,
reset=False,
gain_apply_only=False,
gain_reset=False,
diffgain_apply_only=False,
diffgain_reset=False,
gain_plot_prefix="$STEFCAL_GAIN_PLOT_PREFIX",
gain1_plot_prefix="$STEFCAL_GAIN1_PLOT_PREFIX",
ifrgain_plot_prefix="$STEFCAL_IFRGAIN_PLOT_PREFIX",
diffgain_plot_prefix="$STEFCAL_DIFFGAIN_PLOT_PREFIX",
ifrgain_apply_only=False,
ifrgain_reset=False,
gain_intervals=None,
gain_smoothing=None,
diffgain_intervals=None,
diffgain_smoothing=None,
flag_threshold=None,
calibrate_ifrs="$STEFCAL_CALIBRATE_IFRS",
input_column="$STEFCAL_INPUT_COLUMN",
output_column="$STEFCAL_OUTPUT_COLUMN",
output="CORR_RES",
plotvis="${ms.PLOTVIS}",
dirty=True,
restore=False,
restore_lsm=True,
label=None,
saveconfig="$STEFCAL_SAVE_CONFIG",
plotfail=None,
args=[],
options={},
**kws):
"""Generic function to run a stefcal job.
'section' TDL config file section
'label' will be assigned to the global LABEL for purposes of file naming
'apply_only' if true, will only apply saved solutions rather than re-solve
'{gain,diffgain,ifrgain}_apply_only'
if true, will only apply saved gain/diffgain/IFR gain solutions rather than re-solve
'reset' if true, will reset all saved solutuions prior to starting
'{gain,diffgain,ifrgain}_reset'
if true, will reset saved gain/diffgain/IFR gain solutuions prior to starting
'diffgains' set to a source subset string to solve for diffgains. Set to True to use "=dE"
'diffgain_mode' 'solve-save' to solve & save, 'solve-nosave' to not save, 'apply' to apply only
'diffgain_plot' automatically invoke make_diffgain_plots() if True
'flag_threshold' threshold flaging post-solutions. Give one threshold to flag with --above,
or T1,T2 for --above T1 --fm-above T2
'output' output visibilities ('CORR_DATA','CORR_RES', 'RES' are useful)
'plotvis' if not empty, specifies which output visibilities to plot using plot-ms (see plot.ms.py --help)
'dirty','restore'
'restore_lsm' image output visibilities (passed to imager.make_image above as is)
'plotfail' plotting failure reported via warn or abort. Default is warn, set to abort to abort.
'saveconfig' saves the effective TDL config to file[:section]
'args','options' passed to the stefcal job as is (as a list of arguments and kw=value pairs),
can be used to supply extra TDL options
extra keywords: passed to the stefcal job as kw=value, can be used to supply extra TDL options, thus
overriding settings in the TDL config file. Useful arguments of this kind are e.g.:
stefcal_reset_all=True to remove prior gains solutions.
"""
msname,section,lsm,label,plotvis,calibrate_ifrs, \
gain_plot_prefix,gain1_plot_prefix,ifrgain_plot_prefix,diffgain_plot_prefix,saveconfig,plotfail,input_column,output_column = \
interpolate_locals("msname section lsm label plotvis calibrate_ifrs gain_plot_prefix gain1_plot_prefix ifrgain_plot_prefix diffgain_plot_prefix "
"saveconfig plotfail input_column output_column")
plotfail = plotfail or warn
makedir(v.DESTDIR)
# increment step counter and assign global label
if label is not None:
v.LABEL = str(label)
if type(v.STEP) is int and STEFCAL_STEP_INCR:
v.STEP += STEFCAL_STEP_INCR
# setup stefcal options and run
info("Running stefcal ${step <STEP} ${(<LABEL>)}")
# setup args
args0 = [
"""${ms.MS_TDL} ${ms.CHAN_TDL} ${lsm.LSM_TDL} ms_sel.ms_ifr_subset_str=${ms.IFRS}
ms_sel.input_column=$input_column
ms_sel.output_column=$output_column
stefcal_gain.enabled=1 stefcal_diffgain.enabled=%d %s""" %
((1 if diffgains else 0), STEFCAL_TDLOPTS)
]
if diffgains:
if diffgains is True:
diffgains = "=dE"
args0.append(
"de_subset.subset_enabled=1 de_subset.source_subset=$diffgains")
opts = {
'do_output':
output,
'calibrate_ifrs':
calibrate_ifrs,
'stefcal_gain.mode':
"apply" if apply_only or gain_apply_only else "solve-save",
'stefcal_gain1.mode':
"apply" if apply_only or gain_apply_only else "solve-save",
'stefcal_gain.reset':
int(reset or gain_reset),
'stefcal_diffgain.mode':
"apply" if apply_only or diffgain_apply_only else "solve-save",
'stefcal_diffgain1.mode':
"apply" if apply_only or diffgain_apply_only else "solve-save",
'stefcal_diffgain.reset':
int(reset or diffgain_reset),
'stefcal_diffgain1.reset':
int(reset or diffgain_reset),
'stefcal_ifr_gain_mode':
"apply" if apply_only or ifrgain_apply_only else "solve-save",
'stefcal_ifr_gain_reset':
int(reset or ifrgain_reset),
'stefcal_gain.table':
STEFCAL_GAIN,
'stefcal_gain1.table':
STEFCAL_GAIN1,
'stefcal_diffgain.table':
STEFCAL_DIFFGAIN,
'stefcal_diffgain1.table':
STEFCAL_DIFFGAIN1,
'stefcal_ifr_gain_table':
STEFCAL_IFRGAIN,
'stefcal_visualize':
False
}
# set gain parameters
if gain_smoothing or STEFCAL_GAIN_SMOOTHING or gain_intervals or STEFCAL_GAIN_INTERVALS:
timesmooth, freqsmooth = gain_smoothing or STEFCAL_GAIN_SMOOTHING or (
0, 0)
timeint, freqint = gain_intervals or STEFCAL_GAIN_INTERVALS or (0, 0)
opts['stefcal_gain.timeint'] = 0 if timesmooth else timeint
opts['stefcal_gain.freqint'] = 0 if freqsmooth else freqint
opts['stefcal_gain.timesmooth'] = timesmooth
opts['stefcal_gain.freqsmooth'] = freqsmooth
# set diffgain parameters
if diffgain_smoothing or STEFCAL_DIFFGAIN_SMOOTHING or diffgain_intervals or STEFCAL_DIFFGAIN_INTERVALS:
timesmooth, freqsmooth = diffgain_smoothing or STEFCAL_DIFFGAIN_SMOOTHING or (
0, 0)
timeint, freqint = diffgain_intervals or STEFCAL_DIFFGAIN_INTERVALS or (
0, 0)
opts['stefcal_diffgain.timeint'] = 0 if timesmooth else timeint
opts['stefcal_diffgain.freqint'] = 0 if freqsmooth else freqint
opts['stefcal_diffgain.timesmooth'] = timesmooth
opts['stefcal_diffgain.freqsmooth'] = freqsmooth
# add user-defined args
args0 += list(args)
opts.update(options)
opts.update(kws)
# run the job
mqt.run(STEFCAL_SCRIPT,
STEFCAL_JOBNAME,
section=section,
saveconfig=saveconfig,
args=args0,
options=opts)
# copy gains
try:
if not apply_only:
if os.path.exists(STEFCAL_GAIN) and not gain_apply_only:
std.copy(STEFCAL_GAIN, STEFCAL_GAIN_SAVE)
if gain_plot_prefix:
make_gain_plots(STEFCAL_GAIN_SAVE, prefix=gain_plot_prefix)
if os.path.exists(STEFCAL_GAIN1) and not gain_apply_only:
std.copy(STEFCAL_GAIN1, STEFCAL_GAIN1_SAVE)
if gain_plot_prefix:
make_gain_plots(STEFCAL_GAIN1_SAVE,
prefix=gain_plot_prefix)
if os.path.exists(STEFCAL_DIFFGAIN) and not diffgain_apply_only:
std.copy(STEFCAL_DIFFGAIN, STEFCAL_DIFFGAIN_SAVE)
if diffgain_plot_prefix:
make_diffgain_plots(STEFCAL_DIFFGAIN_SAVE,
prefix=diffgain_plot_prefix)
if os.path.exists(STEFCAL_IFRGAIN) and not ifrgain_apply_only:
std.copy(STEFCAL_IFRGAIN, STEFCAL_IFRGAIN_SAVE)
if ifrgain_plot_prefix:
make_ifrgain_plots(STEFCAL_IFRGAIN_SAVE,
prefix=ifrgain_plot_prefix)
except:
traceback.print_exc()
plotfail("plot routine failed, see exception above")
# post-calibration flagging
if flag_threshold:
if isinstance(flag_threshold, (list, tuple)):
t0, t1 = flag_threshold
else:
t0, t1 = flag_threshold, None
ms.flagms("--above %g" % t0, "-f threshold -c")
if t1:
ms.flagms("--fm-above %g" % t1, "-f fmthreshold -c")
# plot residuals
if plotvis:
try:
info("Plotting visibilities ($plotvis)")
ms.PLOTVIS = plotvis
ms.plotms("-o ${OUTFILE}_${output}${_s<STEP}${_<label}.png")
except:
traceback.print_exc()
plotfail("plot routine failed, see exception above")
# make images
im.make_image(msname,
column=STEFCAL_OUTPUT_COLUMN,
dirty=dirty,
restore=restore,
restore_lsm=restore_lsm)
def stefcal(msname="$MS",
section="$STEFCAL_SECTION",
diffgains=None,
apply_only=False,
gain_apply_only=False,
diffgain_apply_only=False,
ifrgain_apply_only=False,
diffgain_intervals=None,
diffgain_smoothing=None,
flag_threshold=None,
output="CORR_RES",
plotvis="${ms.PLOTVIS}",
dirty=True,
restore=False,
restore_lsm=True,
label=None,
args=[],
options={},
**kws):
"""Generic function to run a stefcal job.
'section' TDL config file section
'label' will be assigned to the global LABEL for purposes of file naming
'apply_only' if true, will only apply saved solutions
'diffgains' set to a source subset string to solve for diffgains. Set to True to use "=dE"
'diffgain_mode' 'solve-save' to solve & save, 'solve-nosave' to not save, 'apply' to apply only
'flag_threshold' threshold flaging post-solutions. Give one threshold to flag with --above,
or T1,T2 for --above T1 --fm-above T2
'output' output visibilities ('CORR_DATA','CORR_RES', 'RES' are useful)
'plotvis' if not empty, specifies which output visibilities to plot using plot-ms (see plot.ms.py --help)
'dirty','restore'
'restore_lsm' image output visibilities (passed to imager.make_image above as is)
'args','options' passed to the stefcal job as is (as a list of arguments and kw=value pairs),
can be used to supply extra TDL options
extra keywords: passed to the stefcal job as kw=value, can be used to supply extra TDL options, thus
overriding settings in the TDL config file. Useful arguments of this kind are e.g.:
stefcal_reset_all=True to remove prior gains solutions.
"""
msname, section, lsm, label, plotvis = interpolate_locals(
"msname section lsm label plotvis")
makedir(v.DESTDIR)
# increment step counter and assign global label
if label is not None:
v.LABEL = str(label)
if type(v.STEP) is int and STEFCAL_STEP_INCR:
v.STEP += STEFCAL_STEP_INCR
# setup stefcal options and run
info("Running stefcal ${step <STEP} ${(<LABEL>)}")
# setup args
args0 = [
"""${ms.MS_TDL} ${ms.CHAN_TDL} ${lsm.LSM_TDL} ms_sel.ms_ifr_subset_str=${ms.IFRS}
ms_sel.output_column=$STEFCAL_OUTPUT_COLUMN
stefcal_gain.enabled=1 stefcal_diffgain.enabled=%d %s""" %
((1 if diffgains else 0), STEFCAL_TDLOPTS)
]
if diffgains:
if diffgains is True:
diffgains = "=dE"
args0.append(
"de_subset.subset_enabled=1 de_subset.source_subset=$diffgains")
opts = {
'do_output':
output,
'stefcal_gain.mode':
"apply" if apply_only or gain_apply_only else "solve-save",
'stefcal_gain1.mode':
"apply" if apply_only or gain_apply_only else "solve-save",
'stefcal_diffgain.mode':
"apply" if apply_only or diffgain_apply_only else "solve-save",
'stefcal_diffgain1.mode':
"apply" if apply_only or diffgain_apply_only else "solve-save",
'stefcal_ifr_gain_mode':
"apply" if apply_only or ifrgain_apply_only else "solve-save",
'stefcal_gain.table':
STEFCAL_GAIN,
'stefcal_gain1.table':
STEFCAL_GAIN1,
'stefcal_diffgain.table':
STEFCAL_DIFFGAIN,
'stefcal_diffgain1.table':
STEFCAL_DIFFGAIN1,
'stefcal_ifr_gain_table':
STEFCAL_IFRGAIN,
'stefcal_visualize':
False
}
timesmooth, freqsmooth = diffgain_smoothing or STEFCAL_DIFFGAIN_SMOOTHING or (
0, 0)
timeint, freqint = diffgain_intervals or STEFCAL_DIFFGAIN_INTERVALS or (0,
0)
opts['stefcal_diffgain.timeint'] = 0 if timesmooth else timeint
opts['stefcal_diffgain.freqint'] = 0 if freqsmooth else freqint
opts['stefcal_diffgain.timesmooth'] = timesmooth
opts['stefcal_diffgain.freqsmooth'] = freqsmooth
# add user-defined args
args0 += list(args)
opts.update(options)
opts.update(kws)
# run the job
mqt.run(STEFCAL_SCRIPT,
STEFCAL_JOBNAME,
section=section,
args=args0,
options=opts)
# copy gains
if not apply_only:
if os.path.exists(STEFCAL_GAIN):
std.copy(STEFCAL_GAIN, STEFCAL_GAIN_SAVE)
if os.path.exists(STEFCAL_GAIN1):
std.copy(STEFCAL_GAIN1, STEFCAL_GAIN1_SAVE)
if os.path.exists(STEFCAL_DIFFGAIN):
std.copy(STEFCAL_DIFFGAIN, STEFCAL_DIFFGAIN_SAVE)
if os.path.exists(STEFCAL_IFRGAIN):
std.copy(STEFCAL_IFRGAIN, STEFCAL_IFRGAIN_SAVE)
# post-calibration flagging
if flag_threshold:
if isinstance(flag_threshold, (list, tuple)):
t0, t1 = flag_threshold
else:
t0, t1 = flag_threshold, None
ms.flagms("--above %g" % t0, "-f threshold -c")
if t1:
ms.flagms("--fm-above %g" % t1, "-f fmthreshold -c")
# plot residuals
if plotvis:
info("Plotting visibilities ($plotvis)")
ms.PLOTVIS = plotvis
ms.plotms("-o ${OUTFILE}_${output}${_s<STEP}${_<label}.png")
# make images
imager.make_image(msname,
column=STEFCAL_OUTPUT_COLUMN,
dirty=dirty,
restore=restore,
restore_lsm=restore_lsm)
def stefcal ( msname="$MS",section="$STEFCAL_SECTION",
diffgains=None,
apply_only=False,
reset=False,
gain_apply_only=False,
gain_reset=False,
diffgain_apply_only=False,
diffgain_reset=False,
gain_plot_prefix="$STEFCAL_GAIN_PLOT_PREFIX",
gain1_plot_prefix="$STEFCAL_GAIN1_PLOT_PREFIX",
ifrgain_plot_prefix="$STEFCAL_IFRGAIN_PLOT_PREFIX",
diffgain_plot_prefix="$STEFCAL_DIFFGAIN_PLOT_PREFIX",
ifrgain_apply_only=False,
ifrgain_reset=False,
gain_intervals=None,gain_smoothing=None,
diffgain_intervals=None,diffgain_smoothing=None,
flag_threshold=None,
calibrate_ifrs="$STEFCAL_CALIBRATE_IFRS",
input_column="$STEFCAL_INPUT_COLUMN",
output_column="$STEFCAL_OUTPUT_COLUMN",
output="CORR_RES",
plotvis="${ms.PLOTVIS}",
dirty=True,restore=False,restore_lsm=True,
label=None,
saveconfig="$STEFCAL_SAVE_CONFIG",
plotfail=None,
args=[],options={},
**kws):
"""Generic function to run a stefcal job.
'section' TDL config file section
'label' will be assigned to the global LABEL for purposes of file naming
'apply_only' if true, will only apply saved solutions rather than re-solve
'{gain,diffgain,ifrgain}_apply_only'
if true, will only apply saved gain/diffgain/IFR gain solutions rather than re-solve
'reset' if true, will reset all saved solutuions prior to starting
'{gain,diffgain,ifrgain}_reset'
if true, will reset saved gain/diffgain/IFR gain solutuions prior to starting
'diffgains' set to a source subset string to solve for diffgains. Set to True to use "=dE"
'diffgain_mode' 'solve-save' to solve & save, 'solve-nosave' to not save, 'apply' to apply only
'diffgain_plot' automatically invoke make_diffgain_plots() if True
'flag_threshold' threshold flaging post-solutions. Give one threshold to flag with --above,
or T1,T2 for --above T1 --fm-above T2
'output' output visibilities ('CORR_DATA','CORR_RES', 'RES' are useful)
'plotvis' if not empty, specifies which output visibilities to plot using plot-ms (see plot.ms.py --help)
'dirty','restore'
'restore_lsm' image output visibilities (passed to imager.make_image above as is)
'plotfail' plotting failure reported via warn or abort. Default is warn, set to abort to abort.
'saveconfig' saves the effective TDL config to file[:section]
'args','options' passed to the stefcal job as is (as a list of arguments and kw=value pairs),
can be used to supply extra TDL options
extra keywords: passed to the stefcal job as kw=value, can be used to supply extra TDL options, thus
overriding settings in the TDL config file. Useful arguments of this kind are e.g.:
stefcal_reset_all=True to remove prior gains solutions.
"""
msname,section,lsm,label,plotvis,calibrate_ifrs, \
gain_plot_prefix,gain1_plot_prefix,ifrgain_plot_prefix,diffgain_plot_prefix,saveconfig,plotfail,input_column,output_column = \
interpolate_locals("msname section lsm label plotvis calibrate_ifrs gain_plot_prefix gain1_plot_prefix ifrgain_plot_prefix diffgain_plot_prefix "
"saveconfig plotfail input_column output_column")
plotfail = plotfail or warn
makedir(v.DESTDIR);
# increment step counter and assign global label
if label is not None:
v.LABEL = str(label);
if type(v.STEP) is int and STEFCAL_STEP_INCR:
v.STEP += STEFCAL_STEP_INCR;
# setup stefcal options and run
info("Running stefcal ${step <STEP} ${(<LABEL>)}");
# setup args
args0 = [ """${ms.MS_TDL} ${ms.CHAN_TDL} ${lsm.LSM_TDL} ms_sel.ms_ifr_subset_str=${ms.IFRS}
ms_sel.input_column=$input_column
ms_sel.output_column=$output_column
stefcal_gain.enabled=1 stefcal_diffgain.enabled=%d %s"""%
((1 if diffgains else 0),STEFCAL_TDLOPTS) ];
if diffgains:
if diffgains is True:
diffgains = "=dE";
args0.append("de_subset.subset_enabled=1 de_subset.source_subset=$diffgains");
opts = {
'do_output': output,
'calibrate_ifrs': calibrate_ifrs,
'stefcal_gain.mode': "apply" if apply_only or gain_apply_only else "solve-save",
'stefcal_gain1.mode': "apply" if apply_only or gain_apply_only else "solve-save",
'stefcal_gain.reset': int(reset or gain_reset),
'stefcal_diffgain.mode': "apply" if apply_only or diffgain_apply_only else "solve-save",
'stefcal_diffgain1.mode': "apply" if apply_only or diffgain_apply_only else "solve-save",
'stefcal_diffgain.reset': int(reset or diffgain_reset),
'stefcal_diffgain1.reset': int(reset or diffgain_reset),
'stefcal_ifr_gain_mode': "apply" if apply_only or ifrgain_apply_only else "solve-save",
'stefcal_ifr_gain_reset': int(reset or ifrgain_reset),
'stefcal_gain.table': STEFCAL_GAIN,
'stefcal_gain1.table': STEFCAL_GAIN1,
'stefcal_diffgain.table': STEFCAL_DIFFGAIN,
'stefcal_diffgain1.table': STEFCAL_DIFFGAIN1,
'stefcal_ifr_gain_table': STEFCAL_IFRGAIN,
'stefcal_visualize': False
}
# set gain parameters
if gain_smoothing or STEFCAL_GAIN_SMOOTHING or gain_intervals or STEFCAL_GAIN_INTERVALS:
timesmooth,freqsmooth = gain_smoothing or STEFCAL_GAIN_SMOOTHING or (0,0);
timeint,freqint = gain_intervals or STEFCAL_GAIN_INTERVALS or (0,0);
opts['stefcal_gain.timeint'] = 0 if timesmooth else timeint;
opts['stefcal_gain.freqint'] = 0 if freqsmooth else freqint;
opts['stefcal_gain.timesmooth'] = timesmooth;
opts['stefcal_gain.freqsmooth'] = freqsmooth;
# set diffgain parameters
if diffgain_smoothing or STEFCAL_DIFFGAIN_SMOOTHING or diffgain_intervals or STEFCAL_DIFFGAIN_INTERVALS:
timesmooth,freqsmooth = diffgain_smoothing or STEFCAL_DIFFGAIN_SMOOTHING or (0,0);
timeint,freqint = diffgain_intervals or STEFCAL_DIFFGAIN_INTERVALS or (0,0);
opts['stefcal_diffgain.timeint'] = 0 if timesmooth else timeint;
opts['stefcal_diffgain.freqint'] = 0 if freqsmooth else freqint;
opts['stefcal_diffgain.timesmooth'] = timesmooth;
opts['stefcal_diffgain.freqsmooth'] = freqsmooth;
# add user-defined args
args0 += list(args);
opts.update(options);
opts.update(kws);
# run the job
mqt.run(STEFCAL_SCRIPT,STEFCAL_JOBNAME,section=section,saveconfig=saveconfig,args=args0,options=opts);
# copy gains
try:
if not apply_only:
if os.path.exists(STEFCAL_GAIN) and not gain_apply_only:
std.copy(STEFCAL_GAIN,STEFCAL_GAIN_SAVE);
if gain_plot_prefix:
make_gain_plots(STEFCAL_GAIN_SAVE,prefix=gain_plot_prefix);
if os.path.exists(STEFCAL_GAIN1) and not gain_apply_only:
std.copy(STEFCAL_GAIN1,STEFCAL_GAIN1_SAVE);
if gain_plot_prefix:
make_gain_plots(STEFCAL_GAIN1_SAVE,prefix=gain_plot_prefix);
if os.path.exists(STEFCAL_DIFFGAIN) and not diffgain_apply_only:
std.copy(STEFCAL_DIFFGAIN,STEFCAL_DIFFGAIN_SAVE);
if diffgain_plot_prefix:
make_diffgain_plots(STEFCAL_DIFFGAIN_SAVE,prefix=diffgain_plot_prefix);
if os.path.exists(STEFCAL_IFRGAIN) and not ifrgain_apply_only:
std.copy(STEFCAL_IFRGAIN,STEFCAL_IFRGAIN_SAVE);
if ifrgain_plot_prefix:
make_ifrgain_plots(STEFCAL_IFRGAIN_SAVE,prefix=ifrgain_plot_prefix);
except:
traceback.print_exc();
plotfail("plot routine failed, see exception above");
# post-calibration flagging
if flag_threshold:
if isinstance(flag_threshold,(list,tuple)):
t0,t1 = flag_threshold;
else:
t0,t1 = flag_threshold,None;
ms.flagms("--above %g"%t0,"-f threshold -c");
if t1:
ms.flagms("--fm-above %g"%t1,"-f fmthreshold -c");
# plot residuals
if plotvis:
try:
info("Plotting visibilities ($plotvis)");
ms.PLOTVIS = plotvis;
ms.plotms("-o ${OUTFILE}_${output}${_s<STEP}${_<label}.png");
except:
traceback.print_exc();
plotfail("plot routine failed, see exception above");
# make images
im.make_image(msname,column=STEFCAL_OUTPUT_COLUMN,dirty=dirty,restore=restore,restore_lsm=restore_lsm);