def _copyfile (filename,dest,missing=abort):
if not exists(filename):
candidates = sorted([ (os.path.getmtime(f),f) for f in glob.glob(filename) ]);
if not candidates:
missing("no match for $filename");
return;
info("$filename specifies %d candidates"%len(candidates));
filename = candidates[-1][1];
info("pre-loading $dest from $filename")
std.copy(filename,dest)
def _copyfile(filename, dest, missing=abort):
if not exists(filename):
candidates = sorted([(os.path.getmtime(f), f)
for f in glob.glob(filename)])
if not candidates:
missing("no match for $filename")
return
info("$filename specifies %d candidates" % len(candidates))
filename = candidates[-1][1]
info("pre-loading $dest from $filename")
std.copy(filename, dest)
def stefcal ( msname="$MS",section="$STEFCAL_SECTION",
diffgains=None,
apply_only=False,
gain_apply_only=False,
diffgain_apply_only=False,
ifrgain_apply_only=False,
diffgain_intervals=None,diffgain_smoothing=None,
flag_threshold=None,
output="CORR_RES",
plotvis="${ms.PLOTVIS}",
dirty=True,restore=False,restore_lsm=True,
label=None,
args=[],options={},
**kws):
"""Generic function to run a stefcal job.
'section' TDL config file section
'label' will be assigned to the global LABEL for purposes of file naming
'apply_only' if true, will only apply saved solutions
'diffgains' set to a source subset string to solve for diffgains. Set to True to use "=dE"
'diffgain_mode' 'solve-save' to solve & save, 'solve-nosave' to not save, 'apply' to apply only
'flag_threshold' threshold flaging post-solutions. Give one threshold to flag with --above,
or T1,T2 for --above T1 --fm-above T2
'output' output visibilities ('CORR_DATA','CORR_RES', 'RES' are useful)
'plotvis' if not empty, specifies which output visibilities to plot using plot-ms (see plot.ms.py --help)
'dirty','restore'
'restore_lsm' image output visibilities (passed to imager.make_image above as is)
'args','options' passed to the stefcal job as is (as a list of arguments and kw=value pairs),
can be used to supply extra TDL options
extra keywords: passed to the stefcal job as kw=value, can be used to supply extra TDL options, thus
overriding settings in the TDL config file. Useful arguments of this kind are e.g.:
stefcal_reset_all=True to remove prior gains solutions.
"""
msname,section,lsm,label,plotvis = interpolate_locals("msname section lsm label plotvis");
makedir(v.DESTDIR);
# increment step counter and assign global label
if label is not None:
v.LABEL = str(label);
if type(v.STEP) is int and STEFCAL_STEP_INCR:
v.STEP += STEFCAL_STEP_INCR;
# setup stefcal options and run
info("Running stefcal ${step <STEP} ${(<LABEL>)}");
# setup args
args0 = [ """${ms.MS_TDL} ${ms.CHAN_TDL} ${lsm.LSM_TDL} ms_sel.ms_ifr_subset_str=${ms.IFRS}
ms_sel.output_column=$STEFCAL_OUTPUT_COLUMN
stefcal_gain.enabled=1 stefcal_diffgain.enabled=%d %s"""%
((1 if diffgains else 0),STEFCAL_TDLOPTS) ];
if diffgains:
if diffgains is True:
diffgains = "=dE";
args0.append("de_subset.subset_enabled=1 de_subset.source_subset=$diffgains");
opts = {
'do_output': output,
'stefcal_gain.mode': "apply" if apply_only or gain_apply_only else "solve-save",
'stefcal_gain1.mode': "apply" if apply_only or gain_apply_only else "solve-save",
'stefcal_diffgain.mode': "apply" if apply_only or diffgain_apply_only else "solve-save",
'stefcal_diffgain1.mode': "apply" if apply_only or diffgain_apply_only else "solve-save",
'stefcal_ifr_gain_mode': "apply" if apply_only or ifrgain_apply_only else "solve-save",
'stefcal_gain.table': STEFCAL_GAIN,
'stefcal_gain1.table': STEFCAL_GAIN1,
'stefcal_diffgain.table': STEFCAL_DIFFGAIN,
'stefcal_diffgain1.table': STEFCAL_DIFFGAIN1,
'stefcal_ifr_gain_table': STEFCAL_IFRGAIN,
'stefcal_visualize': False
}
timesmooth,freqsmooth = diffgain_smoothing or STEFCAL_DIFFGAIN_SMOOTHING or (0,0);
timeint,freqint = diffgain_intervals or STEFCAL_DIFFGAIN_INTERVALS or (0,0);
opts['stefcal_diffgain.timeint'] = 0 if timesmooth else timeint;
opts['stefcal_diffgain.freqint'] = 0 if freqsmooth else freqint;
opts['stefcal_diffgain.timesmooth'] = timesmooth;
opts['stefcal_diffgain.freqsmooth'] = freqsmooth;
# add user-defined args
args0 += list(args);
opts.update(options);
opts.update(kws);
# run the job
mqt.run(STEFCAL_SCRIPT,STEFCAL_JOBNAME,section=section,args=args0,options=opts);
# copy gains
if not apply_only:
if os.path.exists(STEFCAL_GAIN):
std.copy(STEFCAL_GAIN,STEFCAL_GAIN_SAVE);
if os.path.exists(STEFCAL_GAIN1):
std.copy(STEFCAL_GAIN1,STEFCAL_GAIN1_SAVE);
if os.path.exists(STEFCAL_DIFFGAIN):
std.copy(STEFCAL_DIFFGAIN,STEFCAL_DIFFGAIN_SAVE);
if os.path.exists(STEFCAL_IFRGAIN):
std.copy(STEFCAL_IFRGAIN,STEFCAL_IFRGAIN_SAVE);
# post-calibration flagging
if flag_threshold:
if isinstance(flag_threshold,(list,tuple)):
t0,t1 = flag_threshold;
else:
t0,t1 = flag_threshold,None;
ms.flagms("--above %g"%t0,"-f threshold -c");
if t1:
ms.flagms("--fm-above %g"%t1,"-f fmthreshold -c");
# plot residuals
if plotvis:
info("Plotting visibilities ($plotvis)");
ms.PLOTVIS = plotvis;
ms.plotms("-o ${OUTFILE}_${output}${_s<STEP}${_<label}.png");
# make images
imager.make_image(msname,column=STEFCAL_OUTPUT_COLUMN,dirty=dirty,restore=restore,restore_lsm=restore_lsm);
def stefcal(msname="$MS",
section="$STEFCAL_SECTION",
diffgains=None,
apply_only=False,
reset=False,
gain_apply_only=False,
gain_reset=False,
diffgain_apply_only=False,
diffgain_reset=False,
gain_plot_prefix="$STEFCAL_GAIN_PLOT_PREFIX",
gain1_plot_prefix="$STEFCAL_GAIN1_PLOT_PREFIX",
ifrgain_plot_prefix="$STEFCAL_IFRGAIN_PLOT_PREFIX",
diffgain_plot_prefix="$STEFCAL_DIFFGAIN_PLOT_PREFIX",
ifrgain_apply_only=False,
ifrgain_reset=False,
gain_intervals=None,
gain_smoothing=None,
diffgain_intervals=None,
diffgain_smoothing=None,
flag_threshold=None,
calibrate_ifrs="$STEFCAL_CALIBRATE_IFRS",
input_column="$STEFCAL_INPUT_COLUMN",
output_column="$STEFCAL_OUTPUT_COLUMN",
output="CORR_RES",
plotvis="${ms.PLOTVIS}",
dirty=True,
restore=False,
restore_lsm=True,
label=None,
saveconfig="$STEFCAL_SAVE_CONFIG",
plotfail=None,
args=[],
options={},
**kws):
"""Generic function to run a stefcal job.
'section' TDL config file section
'label' will be assigned to the global LABEL for purposes of file naming
'apply_only' if true, will only apply saved solutions rather than re-solve
'{gain,diffgain,ifrgain}_apply_only'
if true, will only apply saved gain/diffgain/IFR gain solutions rather than re-solve
'reset' if true, will reset all saved solutuions prior to starting
'{gain,diffgain,ifrgain}_reset'
if true, will reset saved gain/diffgain/IFR gain solutuions prior to starting
'diffgains' set to a source subset string to solve for diffgains. Set to True to use "=dE"
'diffgain_mode' 'solve-save' to solve & save, 'solve-nosave' to not save, 'apply' to apply only
'diffgain_plot' automatically invoke make_diffgain_plots() if True
'flag_threshold' threshold flaging post-solutions. Give one threshold to flag with --above,
or T1,T2 for --above T1 --fm-above T2
'output' output visibilities ('CORR_DATA','CORR_RES', 'RES' are useful)
'plotvis' if not empty, specifies which output visibilities to plot using plot-ms (see plot.ms.py --help)
'dirty','restore'
'restore_lsm' image output visibilities (passed to imager.make_image above as is)
'plotfail' plotting failure reported via warn or abort. Default is warn, set to abort to abort.
'saveconfig' saves the effective TDL config to file[:section]
'args','options' passed to the stefcal job as is (as a list of arguments and kw=value pairs),
can be used to supply extra TDL options
extra keywords: passed to the stefcal job as kw=value, can be used to supply extra TDL options, thus
overriding settings in the TDL config file. Useful arguments of this kind are e.g.:
stefcal_reset_all=True to remove prior gains solutions.
"""
msname,section,lsm,label,plotvis,calibrate_ifrs, \
gain_plot_prefix,gain1_plot_prefix,ifrgain_plot_prefix,diffgain_plot_prefix,saveconfig,plotfail,input_column,output_column = \
interpolate_locals("msname section lsm label plotvis calibrate_ifrs gain_plot_prefix gain1_plot_prefix ifrgain_plot_prefix diffgain_plot_prefix "
"saveconfig plotfail input_column output_column")
plotfail = plotfail or warn
makedir(v.DESTDIR)
# increment step counter and assign global label
if label is not None:
v.LABEL = str(label)
if type(v.STEP) is int and STEFCAL_STEP_INCR:
v.STEP += STEFCAL_STEP_INCR
# setup stefcal options and run
info("Running stefcal ${step <STEP} ${(<LABEL>)}")
# setup args
args0 = [
"""${ms.MS_TDL} ${ms.CHAN_TDL} ${lsm.LSM_TDL} ms_sel.ms_ifr_subset_str=${ms.IFRS}
ms_sel.input_column=$input_column
ms_sel.output_column=$output_column
stefcal_gain.enabled=1 stefcal_diffgain.enabled=%d %s""" %
((1 if diffgains else 0), STEFCAL_TDLOPTS)
]
if diffgains:
if diffgains is True:
diffgains = "=dE"
args0.append(
"de_subset.subset_enabled=1 de_subset.source_subset=$diffgains")
opts = {
'do_output':
output,
'calibrate_ifrs':
calibrate_ifrs,
'stefcal_gain.mode':
"apply" if apply_only or gain_apply_only else "solve-save",
'stefcal_gain1.mode':
"apply" if apply_only or gain_apply_only else "solve-save",
'stefcal_gain.reset':
int(reset or gain_reset),
'stefcal_diffgain.mode':
"apply" if apply_only or diffgain_apply_only else "solve-save",
'stefcal_diffgain1.mode':
"apply" if apply_only or diffgain_apply_only else "solve-save",
'stefcal_diffgain.reset':
int(reset or diffgain_reset),
'stefcal_diffgain1.reset':
int(reset or diffgain_reset),
'stefcal_ifr_gain_mode':
"apply" if apply_only or ifrgain_apply_only else "solve-save",
'stefcal_ifr_gain_reset':
int(reset or ifrgain_reset),
'stefcal_gain.table':
STEFCAL_GAIN,
'stefcal_gain1.table':
STEFCAL_GAIN1,
'stefcal_diffgain.table':
STEFCAL_DIFFGAIN,
'stefcal_diffgain1.table':
STEFCAL_DIFFGAIN1,
'stefcal_ifr_gain_table':
STEFCAL_IFRGAIN,
'stefcal_visualize':
False
}
# set gain parameters
if gain_smoothing or STEFCAL_GAIN_SMOOTHING or gain_intervals or STEFCAL_GAIN_INTERVALS:
timesmooth, freqsmooth = gain_smoothing or STEFCAL_GAIN_SMOOTHING or (
0, 0)
timeint, freqint = gain_intervals or STEFCAL_GAIN_INTERVALS or (0, 0)
opts['stefcal_gain.timeint'] = 0 if timesmooth else timeint
opts['stefcal_gain.freqint'] = 0 if freqsmooth else freqint
opts['stefcal_gain.timesmooth'] = timesmooth
opts['stefcal_gain.freqsmooth'] = freqsmooth
# set diffgain parameters
if diffgain_smoothing or STEFCAL_DIFFGAIN_SMOOTHING or diffgain_intervals or STEFCAL_DIFFGAIN_INTERVALS:
timesmooth, freqsmooth = diffgain_smoothing or STEFCAL_DIFFGAIN_SMOOTHING or (
0, 0)
timeint, freqint = diffgain_intervals or STEFCAL_DIFFGAIN_INTERVALS or (
0, 0)
opts['stefcal_diffgain.timeint'] = 0 if timesmooth else timeint
opts['stefcal_diffgain.freqint'] = 0 if freqsmooth else freqint
opts['stefcal_diffgain.timesmooth'] = timesmooth
opts['stefcal_diffgain.freqsmooth'] = freqsmooth
# add user-defined args
args0 += list(args)
opts.update(options)
opts.update(kws)
# run the job
mqt.run(STEFCAL_SCRIPT,
STEFCAL_JOBNAME,
section=section,
saveconfig=saveconfig,
args=args0,
options=opts)
# copy gains
try:
if not apply_only:
if os.path.exists(STEFCAL_GAIN) and not gain_apply_only:
std.copy(STEFCAL_GAIN, STEFCAL_GAIN_SAVE)
if gain_plot_prefix:
make_gain_plots(STEFCAL_GAIN_SAVE, prefix=gain_plot_prefix)
if os.path.exists(STEFCAL_GAIN1) and not gain_apply_only:
std.copy(STEFCAL_GAIN1, STEFCAL_GAIN1_SAVE)
if gain_plot_prefix:
make_gain_plots(STEFCAL_GAIN1_SAVE,
prefix=gain_plot_prefix)
if os.path.exists(STEFCAL_DIFFGAIN) and not diffgain_apply_only:
std.copy(STEFCAL_DIFFGAIN, STEFCAL_DIFFGAIN_SAVE)
if diffgain_plot_prefix:
make_diffgain_plots(STEFCAL_DIFFGAIN_SAVE,
prefix=diffgain_plot_prefix)
if os.path.exists(STEFCAL_IFRGAIN) and not ifrgain_apply_only:
std.copy(STEFCAL_IFRGAIN, STEFCAL_IFRGAIN_SAVE)
if ifrgain_plot_prefix:
make_ifrgain_plots(STEFCAL_IFRGAIN_SAVE,
prefix=ifrgain_plot_prefix)
except:
traceback.print_exc()
plotfail("plot routine failed, see exception above")
# post-calibration flagging
if flag_threshold:
if isinstance(flag_threshold, (list, tuple)):
t0, t1 = flag_threshold
else:
t0, t1 = flag_threshold, None
ms.flagms("--above %g" % t0, "-f threshold -c")
if t1:
ms.flagms("--fm-above %g" % t1, "-f fmthreshold -c")
# plot residuals
if plotvis:
try:
info("Plotting visibilities ($plotvis)")
ms.PLOTVIS = plotvis
ms.plotms("-o ${OUTFILE}_${output}${_s<STEP}${_<label}.png")
except:
traceback.print_exc()
plotfail("plot routine failed, see exception above")
# make images
im.make_image(msname,
column=STEFCAL_OUTPUT_COLUMN,
dirty=dirty,
restore=restore,
restore_lsm=restore_lsm)
def azishe():
jdict = readJson(CONFIG)
add = jdict.get("add_to_column", False)
addnoise = jdict.get("addnoise", False)
sefd = jdict.get("sefd", 500)
v.MS = "{:s}/{:s}".format(MSDIR, jdict["msname"])
v.LOG = II("${OUTDIR>/}log-${MS:BASE}-simulation.txt")
column = jdict.get("column", "DATA")
copy = jdict.get("copy_to_CORRECTED_DATA", False)
newcol = jdict.get("custom_data_column", None)
if newcol:
im.argo.addcol(newcol)
if jdict["skymodel"] in [None, False] and addnoise:
noise = compute_vis_noise(sefd)
simnoise(noise, column=column)
if copy and column!="CORRECTED_DATA":
ms.copycol(fromcol=column, tocol="CORRECTED_DATA")
return
for item in ["/input/", "/data/skymodels/"]:
lsmname = "{:s}/{:s}".format(item, jdict["skymodel"])
if os.path.exists(lsmname):
break
v.LSM = II("${lsmname:FILE}")
std.copy(lsmname, LSM)
if jdict.get("recenter", False):
direction = jdict.get("direction", False)
if direction:
ftab = ms.ms(subtable="FIELD")
ra,dec = ftab.getcol("PHASE_DIR")[jdict.get("field_id",0)][0]
direction = "J2000,%frad,%frad"%(ra,dec)
x.sh("tigger-convert --recenter=$direction $LSM -f")
options = {}
if addnoise:
noise = compute_vis_noise(sefd)
options["noise_stddev"] = noise
beam = jdict.get("Ejones", False)
if beam:
options["me.e_enable"] = 1
options["me.p_enable"] = 1
options["me.e_module"] = "Siamese_OMS_pybeams_fits"
options["me.e_advanced"] = 1
options["me.e_all_stations"] = 1
options["pybeams_fits.l_axis"] = jdict.get("beam_l_axis", "L")
options["pybeams_fits.m_axis"] = jdict.get("beam_m_axis", "M")
options["pybeams_fits.filename_pattern"] = "%s/%s"%(INDIR, jdict["beam_files_pattern"])
rms_perr = jdict.get("pointing_accuracy", 0)
# Include pointing errors if needed
if rms_perr:
anttab = ms.ms(subtable="ANTENNA")
NANT = anttab.nrows()
options["me.epe_enable"] = 1
perr = numpy.random.randn(NANT)*rms_perr, numpy.random.randn(NANT)*rms_perr
ll, mm = " ".join( map(str, perr[0]) ), " ".join( map(str, perr[-1]) )
options['oms_pointing_errors.pe_l.values_str'] = "'%s'"%ll
options['oms_pointing_errors.pe_m.values_str'] = "'%s'"%mm
_section = dict(sim = "sim",
add_G = "sim:G")
if jdict.get("gjones", False):
section = "add_G"
else:
section = "sim"
mode = '"add to MS"' if add else '"sim only"'
options["sim_mode"] = mode
options["ms_sel.input_column"] = add
options["ms_sel.output_column"] = column
mqt.msrun(II("${mqt.CATTERY}/Siamese/turbo-sim.py"),
job = '_tdl_job_1_simulate_MS',
section = _section[section],
options = options,
args = ["${lsm.LSM_TDL}"])
if copy and column!="CORRECTED_DATA":
ms.copycol(fromcol=column, tocol="CORRECTED_DATA")
def stefcal(msname="$MS",
section="$STEFCAL_SECTION",
diffgains=None,
apply_only=False,
gain_apply_only=False,
diffgain_apply_only=False,
ifrgain_apply_only=False,
diffgain_intervals=None,
diffgain_smoothing=None,
flag_threshold=None,
output="CORR_RES",
plotvis="${ms.PLOTVIS}",
dirty=True,
restore=False,
restore_lsm=True,
label=None,
args=[],
options={},
**kws):
"""Generic function to run a stefcal job.
'section' TDL config file section
'label' will be assigned to the global LABEL for purposes of file naming
'apply_only' if true, will only apply saved solutions
'diffgains' set to a source subset string to solve for diffgains. Set to True to use "=dE"
'diffgain_mode' 'solve-save' to solve & save, 'solve-nosave' to not save, 'apply' to apply only
'flag_threshold' threshold flaging post-solutions. Give one threshold to flag with --above,
or T1,T2 for --above T1 --fm-above T2
'output' output visibilities ('CORR_DATA','CORR_RES', 'RES' are useful)
'plotvis' if not empty, specifies which output visibilities to plot using plot-ms (see plot.ms.py --help)
'dirty','restore'
'restore_lsm' image output visibilities (passed to imager.make_image above as is)
'args','options' passed to the stefcal job as is (as a list of arguments and kw=value pairs),
can be used to supply extra TDL options
extra keywords: passed to the stefcal job as kw=value, can be used to supply extra TDL options, thus
overriding settings in the TDL config file. Useful arguments of this kind are e.g.:
stefcal_reset_all=True to remove prior gains solutions.
"""
msname, section, lsm, label, plotvis = interpolate_locals(
"msname section lsm label plotvis")
makedir(v.DESTDIR)
# increment step counter and assign global label
if label is not None:
v.LABEL = str(label)
if type(v.STEP) is int and STEFCAL_STEP_INCR:
v.STEP += STEFCAL_STEP_INCR
# setup stefcal options and run
info("Running stefcal ${step <STEP} ${(<LABEL>)}")
# setup args
args0 = [
"""${ms.MS_TDL} ${ms.CHAN_TDL} ${lsm.LSM_TDL} ms_sel.ms_ifr_subset_str=${ms.IFRS}
ms_sel.output_column=$STEFCAL_OUTPUT_COLUMN
stefcal_gain.enabled=1 stefcal_diffgain.enabled=%d %s""" %
((1 if diffgains else 0), STEFCAL_TDLOPTS)
]
if diffgains:
if diffgains is True:
diffgains = "=dE"
args0.append(
"de_subset.subset_enabled=1 de_subset.source_subset=$diffgains")
opts = {
'do_output':
output,
'stefcal_gain.mode':
"apply" if apply_only or gain_apply_only else "solve-save",
'stefcal_gain1.mode':
"apply" if apply_only or gain_apply_only else "solve-save",
'stefcal_diffgain.mode':
"apply" if apply_only or diffgain_apply_only else "solve-save",
'stefcal_diffgain1.mode':
"apply" if apply_only or diffgain_apply_only else "solve-save",
'stefcal_ifr_gain_mode':
"apply" if apply_only or ifrgain_apply_only else "solve-save",
'stefcal_gain.table':
STEFCAL_GAIN,
'stefcal_gain1.table':
STEFCAL_GAIN1,
'stefcal_diffgain.table':
STEFCAL_DIFFGAIN,
'stefcal_diffgain1.table':
STEFCAL_DIFFGAIN1,
'stefcal_ifr_gain_table':
STEFCAL_IFRGAIN,
'stefcal_visualize':
False
}
timesmooth, freqsmooth = diffgain_smoothing or STEFCAL_DIFFGAIN_SMOOTHING or (
0, 0)
timeint, freqint = diffgain_intervals or STEFCAL_DIFFGAIN_INTERVALS or (0,
0)
opts['stefcal_diffgain.timeint'] = 0 if timesmooth else timeint
opts['stefcal_diffgain.freqint'] = 0 if freqsmooth else freqint
opts['stefcal_diffgain.timesmooth'] = timesmooth
opts['stefcal_diffgain.freqsmooth'] = freqsmooth
# add user-defined args
args0 += list(args)
opts.update(options)
opts.update(kws)
# run the job
mqt.run(STEFCAL_SCRIPT,
STEFCAL_JOBNAME,
section=section,
args=args0,
options=opts)
# copy gains
if not apply_only:
if os.path.exists(STEFCAL_GAIN):
std.copy(STEFCAL_GAIN, STEFCAL_GAIN_SAVE)
if os.path.exists(STEFCAL_GAIN1):
std.copy(STEFCAL_GAIN1, STEFCAL_GAIN1_SAVE)
if os.path.exists(STEFCAL_DIFFGAIN):
std.copy(STEFCAL_DIFFGAIN, STEFCAL_DIFFGAIN_SAVE)
if os.path.exists(STEFCAL_IFRGAIN):
std.copy(STEFCAL_IFRGAIN, STEFCAL_IFRGAIN_SAVE)
# post-calibration flagging
if flag_threshold:
if isinstance(flag_threshold, (list, tuple)):
t0, t1 = flag_threshold
else:
t0, t1 = flag_threshold, None
ms.flagms("--above %g" % t0, "-f threshold -c")
if t1:
ms.flagms("--fm-above %g" % t1, "-f fmthreshold -c")
# plot residuals
if plotvis:
info("Plotting visibilities ($plotvis)")
ms.PLOTVIS = plotvis
ms.plotms("-o ${OUTFILE}_${output}${_s<STEP}${_<label}.png")
# make images
imager.make_image(msname,
column=STEFCAL_OUTPUT_COLUMN,
dirty=dirty,
restore=restore,
restore_lsm=restore_lsm)
def stefcal ( msname="$MS",section="$STEFCAL_SECTION",
diffgains=None,
apply_only=False,
reset=False,
gain_apply_only=False,
gain_reset=False,
diffgain_apply_only=False,
diffgain_reset=False,
gain_plot_prefix="$STEFCAL_GAIN_PLOT_PREFIX",
gain1_plot_prefix="$STEFCAL_GAIN1_PLOT_PREFIX",
ifrgain_plot_prefix="$STEFCAL_IFRGAIN_PLOT_PREFIX",
diffgain_plot_prefix="$STEFCAL_DIFFGAIN_PLOT_PREFIX",
ifrgain_apply_only=False,
ifrgain_reset=False,
gain_intervals=None,gain_smoothing=None,
diffgain_intervals=None,diffgain_smoothing=None,
flag_threshold=None,
calibrate_ifrs="$STEFCAL_CALIBRATE_IFRS",
input_column="$STEFCAL_INPUT_COLUMN",
output_column="$STEFCAL_OUTPUT_COLUMN",
output="CORR_RES",
plotvis="${ms.PLOTVIS}",
dirty=True,restore=False,restore_lsm=True,
label=None,
saveconfig="$STEFCAL_SAVE_CONFIG",
plotfail=None,
args=[],options={},
**kws):
"""Generic function to run a stefcal job.
'section' TDL config file section
'label' will be assigned to the global LABEL for purposes of file naming
'apply_only' if true, will only apply saved solutions rather than re-solve
'{gain,diffgain,ifrgain}_apply_only'
if true, will only apply saved gain/diffgain/IFR gain solutions rather than re-solve
'reset' if true, will reset all saved solutuions prior to starting
'{gain,diffgain,ifrgain}_reset'
if true, will reset saved gain/diffgain/IFR gain solutuions prior to starting
'diffgains' set to a source subset string to solve for diffgains. Set to True to use "=dE"
'diffgain_mode' 'solve-save' to solve & save, 'solve-nosave' to not save, 'apply' to apply only
'diffgain_plot' automatically invoke make_diffgain_plots() if True
'flag_threshold' threshold flaging post-solutions. Give one threshold to flag with --above,
or T1,T2 for --above T1 --fm-above T2
'output' output visibilities ('CORR_DATA','CORR_RES', 'RES' are useful)
'plotvis' if not empty, specifies which output visibilities to plot using plot-ms (see plot.ms.py --help)
'dirty','restore'
'restore_lsm' image output visibilities (passed to imager.make_image above as is)
'plotfail' plotting failure reported via warn or abort. Default is warn, set to abort to abort.
'saveconfig' saves the effective TDL config to file[:section]
'args','options' passed to the stefcal job as is (as a list of arguments and kw=value pairs),
can be used to supply extra TDL options
extra keywords: passed to the stefcal job as kw=value, can be used to supply extra TDL options, thus
overriding settings in the TDL config file. Useful arguments of this kind are e.g.:
stefcal_reset_all=True to remove prior gains solutions.
"""
msname,section,lsm,label,plotvis,calibrate_ifrs, \
gain_plot_prefix,gain1_plot_prefix,ifrgain_plot_prefix,diffgain_plot_prefix,saveconfig,plotfail,input_column,output_column = \
interpolate_locals("msname section lsm label plotvis calibrate_ifrs gain_plot_prefix gain1_plot_prefix ifrgain_plot_prefix diffgain_plot_prefix "
"saveconfig plotfail input_column output_column")
plotfail = plotfail or warn
makedir(v.DESTDIR);
# increment step counter and assign global label
if label is not None:
v.LABEL = str(label);
if type(v.STEP) is int and STEFCAL_STEP_INCR:
v.STEP += STEFCAL_STEP_INCR;
# setup stefcal options and run
info("Running stefcal ${step <STEP} ${(<LABEL>)}");
# setup args
args0 = [ """${ms.MS_TDL} ${ms.CHAN_TDL} ${lsm.LSM_TDL} ms_sel.ms_ifr_subset_str=${ms.IFRS}
ms_sel.input_column=$input_column
ms_sel.output_column=$output_column
stefcal_gain.enabled=1 stefcal_diffgain.enabled=%d %s"""%
((1 if diffgains else 0),STEFCAL_TDLOPTS) ];
if diffgains:
if diffgains is True:
diffgains = "=dE";
args0.append("de_subset.subset_enabled=1 de_subset.source_subset=$diffgains");
opts = {
'do_output': output,
'calibrate_ifrs': calibrate_ifrs,
'stefcal_gain.mode': "apply" if apply_only or gain_apply_only else "solve-save",
'stefcal_gain1.mode': "apply" if apply_only or gain_apply_only else "solve-save",
'stefcal_gain.reset': int(reset or gain_reset),
'stefcal_diffgain.mode': "apply" if apply_only or diffgain_apply_only else "solve-save",
'stefcal_diffgain1.mode': "apply" if apply_only or diffgain_apply_only else "solve-save",
'stefcal_diffgain.reset': int(reset or diffgain_reset),
'stefcal_diffgain1.reset': int(reset or diffgain_reset),
'stefcal_ifr_gain_mode': "apply" if apply_only or ifrgain_apply_only else "solve-save",
'stefcal_ifr_gain_reset': int(reset or ifrgain_reset),
'stefcal_gain.table': STEFCAL_GAIN,
'stefcal_gain1.table': STEFCAL_GAIN1,
'stefcal_diffgain.table': STEFCAL_DIFFGAIN,
'stefcal_diffgain1.table': STEFCAL_DIFFGAIN1,
'stefcal_ifr_gain_table': STEFCAL_IFRGAIN,
'stefcal_visualize': False
}
# set gain parameters
if gain_smoothing or STEFCAL_GAIN_SMOOTHING or gain_intervals or STEFCAL_GAIN_INTERVALS:
timesmooth,freqsmooth = gain_smoothing or STEFCAL_GAIN_SMOOTHING or (0,0);
timeint,freqint = gain_intervals or STEFCAL_GAIN_INTERVALS or (0,0);
opts['stefcal_gain.timeint'] = 0 if timesmooth else timeint;
opts['stefcal_gain.freqint'] = 0 if freqsmooth else freqint;
opts['stefcal_gain.timesmooth'] = timesmooth;
opts['stefcal_gain.freqsmooth'] = freqsmooth;
# set diffgain parameters
if diffgain_smoothing or STEFCAL_DIFFGAIN_SMOOTHING or diffgain_intervals or STEFCAL_DIFFGAIN_INTERVALS:
timesmooth,freqsmooth = diffgain_smoothing or STEFCAL_DIFFGAIN_SMOOTHING or (0,0);
timeint,freqint = diffgain_intervals or STEFCAL_DIFFGAIN_INTERVALS or (0,0);
opts['stefcal_diffgain.timeint'] = 0 if timesmooth else timeint;
opts['stefcal_diffgain.freqint'] = 0 if freqsmooth else freqint;
opts['stefcal_diffgain.timesmooth'] = timesmooth;
opts['stefcal_diffgain.freqsmooth'] = freqsmooth;
# add user-defined args
args0 += list(args);
opts.update(options);
opts.update(kws);
# run the job
mqt.run(STEFCAL_SCRIPT,STEFCAL_JOBNAME,section=section,saveconfig=saveconfig,args=args0,options=opts);
# copy gains
try:
if not apply_only:
if os.path.exists(STEFCAL_GAIN) and not gain_apply_only:
std.copy(STEFCAL_GAIN,STEFCAL_GAIN_SAVE);
if gain_plot_prefix:
make_gain_plots(STEFCAL_GAIN_SAVE,prefix=gain_plot_prefix);
if os.path.exists(STEFCAL_GAIN1) and not gain_apply_only:
std.copy(STEFCAL_GAIN1,STEFCAL_GAIN1_SAVE);
if gain_plot_prefix:
make_gain_plots(STEFCAL_GAIN1_SAVE,prefix=gain_plot_prefix);
if os.path.exists(STEFCAL_DIFFGAIN) and not diffgain_apply_only:
std.copy(STEFCAL_DIFFGAIN,STEFCAL_DIFFGAIN_SAVE);
if diffgain_plot_prefix:
make_diffgain_plots(STEFCAL_DIFFGAIN_SAVE,prefix=diffgain_plot_prefix);
if os.path.exists(STEFCAL_IFRGAIN) and not ifrgain_apply_only:
std.copy(STEFCAL_IFRGAIN,STEFCAL_IFRGAIN_SAVE);
if ifrgain_plot_prefix:
make_ifrgain_plots(STEFCAL_IFRGAIN_SAVE,prefix=ifrgain_plot_prefix);
except:
traceback.print_exc();
plotfail("plot routine failed, see exception above");
# post-calibration flagging
if flag_threshold:
if isinstance(flag_threshold,(list,tuple)):
t0,t1 = flag_threshold;
else:
t0,t1 = flag_threshold,None;
ms.flagms("--above %g"%t0,"-f threshold -c");
if t1:
ms.flagms("--fm-above %g"%t1,"-f fmthreshold -c");
# plot residuals
if plotvis:
try:
info("Plotting visibilities ($plotvis)");
ms.PLOTVIS = plotvis;
ms.plotms("-o ${OUTFILE}_${output}${_s<STEP}${_<label}.png");
except:
traceback.print_exc();
plotfail("plot routine failed, see exception above");
# make images
im.make_image(msname,column=STEFCAL_OUTPUT_COLUMN,dirty=dirty,restore=restore,restore_lsm=restore_lsm);
