Beispiel #1
0
def stefcal ( msname="$MS",section="$STEFCAL_SECTION",
              diffgains=None,
              apply_only=False,
              gain_apply_only=False,
              diffgain_apply_only=False,
              ifrgain_apply_only=False,
              diffgain_intervals=None,diffgain_smoothing=None,
              flag_threshold=None,
              output="CORR_RES",
              plotvis="${ms.PLOTVIS}",
              dirty=True,restore=False,restore_lsm=True,
              label=None,
              args=[],options={},
              **kws):
  """Generic function to run a stefcal job.
  
  'section'         TDL config file section
  'label'           will be assigned to the global LABEL for purposes of file naming
  'apply_only'      if true, will only apply saved solutions
  'diffgains'       set to a source subset string to solve for diffgains. Set to True to use "=dE"
  'diffgain_mode'   'solve-save' to solve & save, 'solve-nosave' to not save, 'apply' to apply only
  'flag_threshold'  threshold flaging post-solutions. Give one threshold to flag with --above,
                    or T1,T2 for --above T1 --fm-above T2
  'output'          output visibilities ('CORR_DATA','CORR_RES', 'RES' are useful)
  'plotvis'         if not empty, specifies which output visibilities to plot using plot-ms (see plot.ms.py --help) 
  'dirty','restore' 
  'restore_lsm'     image output visibilities (passed to imager.make_image above as is)
  'args','options'  passed to the stefcal job as is (as a list of arguments and kw=value pairs), 
                    can be used to supply extra TDL options
  extra keywords:   passed to the stefcal job as kw=value, can be used to supply extra TDL options, thus
                    overriding settings in the TDL config file. Useful arguments of this kind are e.g.:
                    stefcal_reset_all=True to remove prior gains solutions.
  """
  msname,section,lsm,label,plotvis = interpolate_locals("msname section lsm label plotvis");
  
  makedir(v.DESTDIR);
  
  # increment step counter and assign global label
  
  if label is not None:
    v.LABEL = str(label);
  if type(v.STEP) is int and STEFCAL_STEP_INCR:
    v.STEP += STEFCAL_STEP_INCR;

  # setup stefcal options and run 
  info("Running stefcal ${step <STEP} ${(<LABEL>)}");
  # setup args
  args0 = [ """${ms.MS_TDL} ${ms.CHAN_TDL} ${lsm.LSM_TDL} ms_sel.ms_ifr_subset_str=${ms.IFRS} 
    ms_sel.output_column=$STEFCAL_OUTPUT_COLUMN
    stefcal_gain.enabled=1 stefcal_diffgain.enabled=%d %s"""%
    ((1 if diffgains else 0),STEFCAL_TDLOPTS) ];
  if diffgains:
    if diffgains is True:
      diffgains = "=dE";
    args0.append("de_subset.subset_enabled=1 de_subset.source_subset=$diffgains"); 
  opts = {
    'do_output': output,
    'stefcal_gain.mode': "apply" if apply_only or gain_apply_only else "solve-save",
    'stefcal_gain1.mode': "apply" if apply_only  or gain_apply_only else "solve-save",
    'stefcal_diffgain.mode': "apply" if apply_only or diffgain_apply_only else "solve-save",
    'stefcal_diffgain1.mode': "apply" if apply_only or diffgain_apply_only  else "solve-save",
    'stefcal_ifr_gain_mode': "apply" if apply_only or ifrgain_apply_only else "solve-save",
    'stefcal_gain.table': STEFCAL_GAIN,
    'stefcal_gain1.table': STEFCAL_GAIN1,
    'stefcal_diffgain.table': STEFCAL_DIFFGAIN,
    'stefcal_diffgain1.table': STEFCAL_DIFFGAIN1,
    'stefcal_ifr_gain_table': STEFCAL_IFRGAIN,
    'stefcal_visualize': False
  }
  timesmooth,freqsmooth = diffgain_smoothing or STEFCAL_DIFFGAIN_SMOOTHING or (0,0);
  timeint,freqint = diffgain_intervals or STEFCAL_DIFFGAIN_INTERVALS or (0,0);
  opts['stefcal_diffgain.timeint'] = 0 if timesmooth else timeint;
  opts['stefcal_diffgain.freqint'] = 0 if freqsmooth else freqint;
  opts['stefcal_diffgain.timesmooth'] = timesmooth;
  opts['stefcal_diffgain.freqsmooth'] = freqsmooth;

  # add user-defined args
  args0 += list(args);
  opts.update(options);
  opts.update(kws);
  # run the job
  mqt.run(STEFCAL_SCRIPT,STEFCAL_JOBNAME,section=section,args=args0,options=opts);
  
  # copy gains
  if not apply_only:
    if os.path.exists(STEFCAL_GAIN):
      std.copy(STEFCAL_GAIN,STEFCAL_GAIN_SAVE);
    if os.path.exists(STEFCAL_GAIN1):
      std.copy(STEFCAL_GAIN1,STEFCAL_GAIN1_SAVE);
    if os.path.exists(STEFCAL_DIFFGAIN):
      std.copy(STEFCAL_DIFFGAIN,STEFCAL_DIFFGAIN_SAVE);
    if os.path.exists(STEFCAL_IFRGAIN):
      std.copy(STEFCAL_IFRGAIN,STEFCAL_IFRGAIN_SAVE);
    
  # post-calibration flagging
  if flag_threshold:
    if isinstance(flag_threshold,(list,tuple)):
      t0,t1 = flag_threshold;
    else:
      t0,t1 = flag_threshold,None;
    ms.flagms("--above %g"%t0,"-f threshold -c");
    if t1:
      ms.flagms("--fm-above %g"%t1,"-f fmthreshold -c");

  # plot residuals
  if plotvis:
    info("Plotting visibilities ($plotvis)");
    ms.PLOTVIS = plotvis;
    ms.plotms("-o ${OUTFILE}_${output}${_s<STEP}${_<label}.png");
    
  # make images
  imager.make_image(msname,column=STEFCAL_OUTPUT_COLUMN,dirty=dirty,restore=restore,restore_lsm=restore_lsm);
def stefcal(msname="$MS",
            section="$STEFCAL_SECTION",
            diffgains=None,
            apply_only=False,
            gain_apply_only=False,
            diffgain_apply_only=False,
            ifrgain_apply_only=False,
            diffgain_intervals=None,
            diffgain_smoothing=None,
            flag_threshold=None,
            output="CORR_RES",
            plotvis="${ms.PLOTVIS}",
            dirty=True,
            restore=False,
            restore_lsm=True,
            label=None,
            args=[],
            options={},
            **kws):
    """Generic function to run a stefcal job.
  
  'section'         TDL config file section
  'label'           will be assigned to the global LABEL for purposes of file naming
  'apply_only'      if true, will only apply saved solutions
  'diffgains'       set to a source subset string to solve for diffgains. Set to True to use "=dE"
  'diffgain_mode'   'solve-save' to solve & save, 'solve-nosave' to not save, 'apply' to apply only
  'flag_threshold'  threshold flaging post-solutions. Give one threshold to flag with --above,
                    or T1,T2 for --above T1 --fm-above T2
  'output'          output visibilities ('CORR_DATA','CORR_RES', 'RES' are useful)
  'plotvis'         if not empty, specifies which output visibilities to plot using plot-ms (see plot.ms.py --help) 
  'dirty','restore' 
  'restore_lsm'     image output visibilities (passed to imager.make_image above as is)
  'args','options'  passed to the stefcal job as is (as a list of arguments and kw=value pairs), 
                    can be used to supply extra TDL options
  extra keywords:   passed to the stefcal job as kw=value, can be used to supply extra TDL options, thus
                    overriding settings in the TDL config file. Useful arguments of this kind are e.g.:
                    stefcal_reset_all=True to remove prior gains solutions.
  """
    msname, section, lsm, label, plotvis = interpolate_locals(
        "msname section lsm label plotvis")

    makedir(v.DESTDIR)

    # increment step counter and assign global label

    if label is not None:
        v.LABEL = str(label)
    if type(v.STEP) is int and STEFCAL_STEP_INCR:
        v.STEP += STEFCAL_STEP_INCR

    # setup stefcal options and run
    info("Running stefcal ${step <STEP} ${(<LABEL>)}")
    # setup args
    args0 = [
        """${ms.MS_TDL} ${ms.CHAN_TDL} ${lsm.LSM_TDL} ms_sel.ms_ifr_subset_str=${ms.IFRS} 
    ms_sel.output_column=$STEFCAL_OUTPUT_COLUMN
    stefcal_gain.enabled=1 stefcal_diffgain.enabled=%d %s""" %
        ((1 if diffgains else 0), STEFCAL_TDLOPTS)
    ]
    if diffgains:
        if diffgains is True:
            diffgains = "=dE"
        args0.append(
            "de_subset.subset_enabled=1 de_subset.source_subset=$diffgains")
    opts = {
        'do_output':
        output,
        'stefcal_gain.mode':
        "apply" if apply_only or gain_apply_only else "solve-save",
        'stefcal_gain1.mode':
        "apply" if apply_only or gain_apply_only else "solve-save",
        'stefcal_diffgain.mode':
        "apply" if apply_only or diffgain_apply_only else "solve-save",
        'stefcal_diffgain1.mode':
        "apply" if apply_only or diffgain_apply_only else "solve-save",
        'stefcal_ifr_gain_mode':
        "apply" if apply_only or ifrgain_apply_only else "solve-save",
        'stefcal_gain.table':
        STEFCAL_GAIN,
        'stefcal_gain1.table':
        STEFCAL_GAIN1,
        'stefcal_diffgain.table':
        STEFCAL_DIFFGAIN,
        'stefcal_diffgain1.table':
        STEFCAL_DIFFGAIN1,
        'stefcal_ifr_gain_table':
        STEFCAL_IFRGAIN,
        'stefcal_visualize':
        False
    }
    timesmooth, freqsmooth = diffgain_smoothing or STEFCAL_DIFFGAIN_SMOOTHING or (
        0, 0)
    timeint, freqint = diffgain_intervals or STEFCAL_DIFFGAIN_INTERVALS or (0,
                                                                            0)
    opts['stefcal_diffgain.timeint'] = 0 if timesmooth else timeint
    opts['stefcal_diffgain.freqint'] = 0 if freqsmooth else freqint
    opts['stefcal_diffgain.timesmooth'] = timesmooth
    opts['stefcal_diffgain.freqsmooth'] = freqsmooth

    # add user-defined args
    args0 += list(args)
    opts.update(options)
    opts.update(kws)
    # run the job
    mqt.run(STEFCAL_SCRIPT,
            STEFCAL_JOBNAME,
            section=section,
            args=args0,
            options=opts)

    # copy gains
    if not apply_only:
        if os.path.exists(STEFCAL_GAIN):
            std.copy(STEFCAL_GAIN, STEFCAL_GAIN_SAVE)
        if os.path.exists(STEFCAL_GAIN1):
            std.copy(STEFCAL_GAIN1, STEFCAL_GAIN1_SAVE)
        if os.path.exists(STEFCAL_DIFFGAIN):
            std.copy(STEFCAL_DIFFGAIN, STEFCAL_DIFFGAIN_SAVE)
        if os.path.exists(STEFCAL_IFRGAIN):
            std.copy(STEFCAL_IFRGAIN, STEFCAL_IFRGAIN_SAVE)

    # post-calibration flagging
    if flag_threshold:
        if isinstance(flag_threshold, (list, tuple)):
            t0, t1 = flag_threshold
        else:
            t0, t1 = flag_threshold, None
        ms.flagms("--above %g" % t0, "-f threshold -c")
        if t1:
            ms.flagms("--fm-above %g" % t1, "-f fmthreshold -c")

    # plot residuals
    if plotvis:
        info("Plotting visibilities ($plotvis)")
        ms.PLOTVIS = plotvis
        ms.plotms("-o ${OUTFILE}_${output}${_s<STEP}${_<label}.png")

    # make images
    imager.make_image(msname,
                      column=STEFCAL_OUTPUT_COLUMN,
                      dirty=dirty,
                      restore=restore,
                      restore_lsm=restore_lsm)
Beispiel #3
0
def stefcal(msname="$MS",
            section="$STEFCAL_SECTION",
            diffgains=None,
            apply_only=False,
            reset=False,
            gain_apply_only=False,
            gain_reset=False,
            diffgain_apply_only=False,
            diffgain_reset=False,
            gain_plot_prefix="$STEFCAL_GAIN_PLOT_PREFIX",
            gain1_plot_prefix="$STEFCAL_GAIN1_PLOT_PREFIX",
            ifrgain_plot_prefix="$STEFCAL_IFRGAIN_PLOT_PREFIX",
            diffgain_plot_prefix="$STEFCAL_DIFFGAIN_PLOT_PREFIX",
            ifrgain_apply_only=False,
            ifrgain_reset=False,
            gain_intervals=None,
            gain_smoothing=None,
            diffgain_intervals=None,
            diffgain_smoothing=None,
            flag_threshold=None,
            calibrate_ifrs="$STEFCAL_CALIBRATE_IFRS",
            input_column="$STEFCAL_INPUT_COLUMN",
            output_column="$STEFCAL_OUTPUT_COLUMN",
            output="CORR_RES",
            plotvis="${ms.PLOTVIS}",
            dirty=True,
            restore=False,
            restore_lsm=True,
            label=None,
            saveconfig="$STEFCAL_SAVE_CONFIG",
            plotfail=None,
            args=[],
            options={},
            **kws):
    """Generic function to run a stefcal job.
  
  'section'         TDL config file section
  'label'           will be assigned to the global LABEL for purposes of file naming
  'apply_only'      if true, will only apply saved solutions rather than re-solve
  '{gain,diffgain,ifrgain}_apply_only'      
                    if true, will only apply saved gain/diffgain/IFR gain solutions rather than re-solve
  'reset'           if true, will reset all saved solutuions prior to starting
  '{gain,diffgain,ifrgain}_reset'      
                    if true, will reset saved gain/diffgain/IFR gain solutuions prior to starting
  'diffgains'       set to a source subset string to solve for diffgains. Set to True to use "=dE"
  'diffgain_mode'   'solve-save' to solve & save, 'solve-nosave' to not save, 'apply' to apply only
  'diffgain_plot'   automatically invoke make_diffgain_plots() if True
  'flag_threshold'  threshold flaging post-solutions. Give one threshold to flag with --above,
                    or T1,T2 for --above T1 --fm-above T2
  'output'          output visibilities ('CORR_DATA','CORR_RES', 'RES' are useful)
  'plotvis'         if not empty, specifies which output visibilities to plot using plot-ms (see plot.ms.py --help) 
  'dirty','restore' 
  'restore_lsm'     image output visibilities (passed to imager.make_image above as is)
  'plotfail'        plotting failure reported via warn or abort. Default is warn, set to abort to abort. 
  'saveconfig'      saves the effective TDL config to file[:section]
  'args','options'  passed to the stefcal job as is (as a list of arguments and kw=value pairs), 
                    can be used to supply extra TDL options
  extra keywords:   passed to the stefcal job as kw=value, can be used to supply extra TDL options, thus
                    overriding settings in the TDL config file. Useful arguments of this kind are e.g.:
                    stefcal_reset_all=True to remove prior gains solutions.
  """
    msname,section,lsm,label,plotvis,calibrate_ifrs, \
      gain_plot_prefix,gain1_plot_prefix,ifrgain_plot_prefix,diffgain_plot_prefix,saveconfig,plotfail,input_column,output_column = \
      interpolate_locals("msname section lsm label plotvis calibrate_ifrs gain_plot_prefix gain1_plot_prefix ifrgain_plot_prefix diffgain_plot_prefix "
        "saveconfig plotfail input_column output_column")

    plotfail = plotfail or warn
    makedir(v.DESTDIR)

    # increment step counter and assign global label

    if label is not None:
        v.LABEL = str(label)
    if type(v.STEP) is int and STEFCAL_STEP_INCR:
        v.STEP += STEFCAL_STEP_INCR

    # setup stefcal options and run
    info("Running stefcal ${step <STEP} ${(<LABEL>)}")
    # setup args
    args0 = [
        """${ms.MS_TDL} ${ms.CHAN_TDL} ${lsm.LSM_TDL} ms_sel.ms_ifr_subset_str=${ms.IFRS} 
    ms_sel.input_column=$input_column
    ms_sel.output_column=$output_column
    stefcal_gain.enabled=1 stefcal_diffgain.enabled=%d %s""" %
        ((1 if diffgains else 0), STEFCAL_TDLOPTS)
    ]
    if diffgains:
        if diffgains is True:
            diffgains = "=dE"
        args0.append(
            "de_subset.subset_enabled=1 de_subset.source_subset=$diffgains")
    opts = {
        'do_output':
        output,
        'calibrate_ifrs':
        calibrate_ifrs,
        'stefcal_gain.mode':
        "apply" if apply_only or gain_apply_only else "solve-save",
        'stefcal_gain1.mode':
        "apply" if apply_only or gain_apply_only else "solve-save",
        'stefcal_gain.reset':
        int(reset or gain_reset),
        'stefcal_diffgain.mode':
        "apply" if apply_only or diffgain_apply_only else "solve-save",
        'stefcal_diffgain1.mode':
        "apply" if apply_only or diffgain_apply_only else "solve-save",
        'stefcal_diffgain.reset':
        int(reset or diffgain_reset),
        'stefcal_diffgain1.reset':
        int(reset or diffgain_reset),
        'stefcal_ifr_gain_mode':
        "apply" if apply_only or ifrgain_apply_only else "solve-save",
        'stefcal_ifr_gain_reset':
        int(reset or ifrgain_reset),
        'stefcal_gain.table':
        STEFCAL_GAIN,
        'stefcal_gain1.table':
        STEFCAL_GAIN1,
        'stefcal_diffgain.table':
        STEFCAL_DIFFGAIN,
        'stefcal_diffgain1.table':
        STEFCAL_DIFFGAIN1,
        'stefcal_ifr_gain_table':
        STEFCAL_IFRGAIN,
        'stefcal_visualize':
        False
    }
    # set gain parameters
    if gain_smoothing or STEFCAL_GAIN_SMOOTHING or gain_intervals or STEFCAL_GAIN_INTERVALS:
        timesmooth, freqsmooth = gain_smoothing or STEFCAL_GAIN_SMOOTHING or (
            0, 0)
        timeint, freqint = gain_intervals or STEFCAL_GAIN_INTERVALS or (0, 0)
        opts['stefcal_gain.timeint'] = 0 if timesmooth else timeint
        opts['stefcal_gain.freqint'] = 0 if freqsmooth else freqint
        opts['stefcal_gain.timesmooth'] = timesmooth
        opts['stefcal_gain.freqsmooth'] = freqsmooth
    # set diffgain parameters
    if diffgain_smoothing or STEFCAL_DIFFGAIN_SMOOTHING or diffgain_intervals or STEFCAL_DIFFGAIN_INTERVALS:
        timesmooth, freqsmooth = diffgain_smoothing or STEFCAL_DIFFGAIN_SMOOTHING or (
            0, 0)
        timeint, freqint = diffgain_intervals or STEFCAL_DIFFGAIN_INTERVALS or (
            0, 0)
        opts['stefcal_diffgain.timeint'] = 0 if timesmooth else timeint
        opts['stefcal_diffgain.freqint'] = 0 if freqsmooth else freqint
        opts['stefcal_diffgain.timesmooth'] = timesmooth
        opts['stefcal_diffgain.freqsmooth'] = freqsmooth

    # add user-defined args
    args0 += list(args)
    opts.update(options)
    opts.update(kws)
    # run the job
    mqt.run(STEFCAL_SCRIPT,
            STEFCAL_JOBNAME,
            section=section,
            saveconfig=saveconfig,
            args=args0,
            options=opts)

    # copy gains
    try:
        if not apply_only:
            if os.path.exists(STEFCAL_GAIN) and not gain_apply_only:
                std.copy(STEFCAL_GAIN, STEFCAL_GAIN_SAVE)
                if gain_plot_prefix:
                    make_gain_plots(STEFCAL_GAIN_SAVE, prefix=gain_plot_prefix)
            if os.path.exists(STEFCAL_GAIN1) and not gain_apply_only:
                std.copy(STEFCAL_GAIN1, STEFCAL_GAIN1_SAVE)
                if gain_plot_prefix:
                    make_gain_plots(STEFCAL_GAIN1_SAVE,
                                    prefix=gain_plot_prefix)
            if os.path.exists(STEFCAL_DIFFGAIN) and not diffgain_apply_only:
                std.copy(STEFCAL_DIFFGAIN, STEFCAL_DIFFGAIN_SAVE)
                if diffgain_plot_prefix:
                    make_diffgain_plots(STEFCAL_DIFFGAIN_SAVE,
                                        prefix=diffgain_plot_prefix)
            if os.path.exists(STEFCAL_IFRGAIN) and not ifrgain_apply_only:
                std.copy(STEFCAL_IFRGAIN, STEFCAL_IFRGAIN_SAVE)
                if ifrgain_plot_prefix:
                    make_ifrgain_plots(STEFCAL_IFRGAIN_SAVE,
                                       prefix=ifrgain_plot_prefix)
    except:
        traceback.print_exc()
        plotfail("plot routine failed, see exception above")
    # post-calibration flagging
    if flag_threshold:
        if isinstance(flag_threshold, (list, tuple)):
            t0, t1 = flag_threshold
        else:
            t0, t1 = flag_threshold, None
        ms.flagms("--above %g" % t0, "-f threshold -c")
        if t1:
            ms.flagms("--fm-above %g" % t1, "-f fmthreshold -c")

    # plot residuals
    if plotvis:
        try:
            info("Plotting visibilities ($plotvis)")
            ms.PLOTVIS = plotvis
            ms.plotms("-o ${OUTFILE}_${output}${_s<STEP}${_<label}.png")
        except:
            traceback.print_exc()
            plotfail("plot routine failed, see exception above")

    # make images
    im.make_image(msname,
                  column=STEFCAL_OUTPUT_COLUMN,
                  dirty=dirty,
                  restore=restore,
                  restore_lsm=restore_lsm)
Beispiel #4
0
def stefcal ( msname="$MS",section="$STEFCAL_SECTION",
              diffgains=None,
              apply_only=False,
              reset=False,
              gain_apply_only=False,
              gain_reset=False,
              diffgain_apply_only=False,
              diffgain_reset=False,
              gain_plot_prefix="$STEFCAL_GAIN_PLOT_PREFIX",
              gain1_plot_prefix="$STEFCAL_GAIN1_PLOT_PREFIX",
              ifrgain_plot_prefix="$STEFCAL_IFRGAIN_PLOT_PREFIX",
              diffgain_plot_prefix="$STEFCAL_DIFFGAIN_PLOT_PREFIX",
              ifrgain_apply_only=False,
              ifrgain_reset=False,
              gain_intervals=None,gain_smoothing=None,
              diffgain_intervals=None,diffgain_smoothing=None,
              flag_threshold=None,
              calibrate_ifrs="$STEFCAL_CALIBRATE_IFRS",
              input_column="$STEFCAL_INPUT_COLUMN",
              output_column="$STEFCAL_OUTPUT_COLUMN",
              output="CORR_RES",
              plotvis="${ms.PLOTVIS}",
              dirty=True,restore=False,restore_lsm=True,
              label=None,
              saveconfig="$STEFCAL_SAVE_CONFIG",
              plotfail=None,
              args=[],options={},
              **kws):
  """Generic function to run a stefcal job.
  
  'section'         TDL config file section
  'label'           will be assigned to the global LABEL for purposes of file naming
  'apply_only'      if true, will only apply saved solutions rather than re-solve
  '{gain,diffgain,ifrgain}_apply_only'      
                    if true, will only apply saved gain/diffgain/IFR gain solutions rather than re-solve
  'reset'           if true, will reset all saved solutuions prior to starting
  '{gain,diffgain,ifrgain}_reset'      
                    if true, will reset saved gain/diffgain/IFR gain solutuions prior to starting
  'diffgains'       set to a source subset string to solve for diffgains. Set to True to use "=dE"
  'diffgain_mode'   'solve-save' to solve & save, 'solve-nosave' to not save, 'apply' to apply only
  'diffgain_plot'   automatically invoke make_diffgain_plots() if True
  'flag_threshold'  threshold flaging post-solutions. Give one threshold to flag with --above,
                    or T1,T2 for --above T1 --fm-above T2
  'output'          output visibilities ('CORR_DATA','CORR_RES', 'RES' are useful)
  'plotvis'         if not empty, specifies which output visibilities to plot using plot-ms (see plot.ms.py --help) 
  'dirty','restore' 
  'restore_lsm'     image output visibilities (passed to imager.make_image above as is)
  'plotfail'        plotting failure reported via warn or abort. Default is warn, set to abort to abort. 
  'saveconfig'      saves the effective TDL config to file[:section]
  'args','options'  passed to the stefcal job as is (as a list of arguments and kw=value pairs), 
                    can be used to supply extra TDL options
  extra keywords:   passed to the stefcal job as kw=value, can be used to supply extra TDL options, thus
                    overriding settings in the TDL config file. Useful arguments of this kind are e.g.:
                    stefcal_reset_all=True to remove prior gains solutions.
  """
  msname,section,lsm,label,plotvis,calibrate_ifrs, \
    gain_plot_prefix,gain1_plot_prefix,ifrgain_plot_prefix,diffgain_plot_prefix,saveconfig,plotfail,input_column,output_column = \
    interpolate_locals("msname section lsm label plotvis calibrate_ifrs gain_plot_prefix gain1_plot_prefix ifrgain_plot_prefix diffgain_plot_prefix "
      "saveconfig plotfail input_column output_column")
  
  plotfail = plotfail or warn
  makedir(v.DESTDIR);
  
  # increment step counter and assign global label
  
  if label is not None:
    v.LABEL = str(label);
  if type(v.STEP) is int and STEFCAL_STEP_INCR:
    v.STEP += STEFCAL_STEP_INCR;

  # setup stefcal options and run 
  info("Running stefcal ${step <STEP} ${(<LABEL>)}");
  # setup args
  args0 = [ """${ms.MS_TDL} ${ms.CHAN_TDL} ${lsm.LSM_TDL} ms_sel.ms_ifr_subset_str=${ms.IFRS} 
    ms_sel.input_column=$input_column
    ms_sel.output_column=$output_column
    stefcal_gain.enabled=1 stefcal_diffgain.enabled=%d %s"""%
    ((1 if diffgains else 0),STEFCAL_TDLOPTS) ];
  if diffgains:
    if diffgains is True:
      diffgains = "=dE";
    args0.append("de_subset.subset_enabled=1 de_subset.source_subset=$diffgains"); 
  opts = {
    'do_output': output,
    'calibrate_ifrs': calibrate_ifrs,
    'stefcal_gain.mode': "apply" if apply_only or gain_apply_only else "solve-save",
    'stefcal_gain1.mode': "apply" if apply_only  or gain_apply_only else "solve-save",
    'stefcal_gain.reset': int(reset or gain_reset),
    'stefcal_diffgain.mode': "apply" if apply_only or diffgain_apply_only else "solve-save",
    'stefcal_diffgain1.mode': "apply" if apply_only or diffgain_apply_only  else "solve-save",
    'stefcal_diffgain.reset': int(reset or diffgain_reset),
    'stefcal_diffgain1.reset': int(reset or diffgain_reset),
    'stefcal_ifr_gain_mode': "apply" if apply_only or ifrgain_apply_only else "solve-save",
    'stefcal_ifr_gain_reset': int(reset or ifrgain_reset),
    'stefcal_gain.table': STEFCAL_GAIN,
    'stefcal_gain1.table': STEFCAL_GAIN1,
    'stefcal_diffgain.table': STEFCAL_DIFFGAIN,
    'stefcal_diffgain1.table': STEFCAL_DIFFGAIN1,
    'stefcal_ifr_gain_table': STEFCAL_IFRGAIN,
    'stefcal_visualize': False
  }
  # set gain parameters
  if gain_smoothing or STEFCAL_GAIN_SMOOTHING or gain_intervals or STEFCAL_GAIN_INTERVALS:
      timesmooth,freqsmooth = gain_smoothing or STEFCAL_GAIN_SMOOTHING or (0,0);
      timeint,freqint = gain_intervals or STEFCAL_GAIN_INTERVALS or (0,0);
      opts['stefcal_gain.timeint'] = 0 if timesmooth else timeint;
      opts['stefcal_gain.freqint'] = 0 if freqsmooth else freqint;
      opts['stefcal_gain.timesmooth'] = timesmooth;
      opts['stefcal_gain.freqsmooth'] = freqsmooth;
  # set diffgain parameters
  if diffgain_smoothing or STEFCAL_DIFFGAIN_SMOOTHING or diffgain_intervals or STEFCAL_DIFFGAIN_INTERVALS:
      timesmooth,freqsmooth = diffgain_smoothing or STEFCAL_DIFFGAIN_SMOOTHING or (0,0);
      timeint,freqint = diffgain_intervals or STEFCAL_DIFFGAIN_INTERVALS or (0,0);
      opts['stefcal_diffgain.timeint'] = 0 if timesmooth else timeint;
      opts['stefcal_diffgain.freqint'] = 0 if freqsmooth else freqint;
      opts['stefcal_diffgain.timesmooth'] = timesmooth;
      opts['stefcal_diffgain.freqsmooth'] = freqsmooth;

  # add user-defined args
  args0 += list(args);
  opts.update(options);
  opts.update(kws);
  # run the job
  mqt.run(STEFCAL_SCRIPT,STEFCAL_JOBNAME,section=section,saveconfig=saveconfig,args=args0,options=opts);
  
  # copy gains
  try:
    if not apply_only:
      if os.path.exists(STEFCAL_GAIN) and not gain_apply_only:
        std.copy(STEFCAL_GAIN,STEFCAL_GAIN_SAVE);
        if gain_plot_prefix:
          make_gain_plots(STEFCAL_GAIN_SAVE,prefix=gain_plot_prefix);
      if os.path.exists(STEFCAL_GAIN1) and not gain_apply_only:
        std.copy(STEFCAL_GAIN1,STEFCAL_GAIN1_SAVE);
        if gain_plot_prefix:
          make_gain_plots(STEFCAL_GAIN1_SAVE,prefix=gain_plot_prefix);
      if os.path.exists(STEFCAL_DIFFGAIN) and not diffgain_apply_only:
        std.copy(STEFCAL_DIFFGAIN,STEFCAL_DIFFGAIN_SAVE);
        if diffgain_plot_prefix:
          make_diffgain_plots(STEFCAL_DIFFGAIN_SAVE,prefix=diffgain_plot_prefix);
      if os.path.exists(STEFCAL_IFRGAIN) and not ifrgain_apply_only:
        std.copy(STEFCAL_IFRGAIN,STEFCAL_IFRGAIN_SAVE);
        if ifrgain_plot_prefix:
          make_ifrgain_plots(STEFCAL_IFRGAIN_SAVE,prefix=ifrgain_plot_prefix);
  except:
    traceback.print_exc();
    plotfail("plot routine failed, see exception above");                  
  # post-calibration flagging
  if flag_threshold:
    if isinstance(flag_threshold,(list,tuple)):
      t0,t1 = flag_threshold;
    else:
      t0,t1 = flag_threshold,None;
    ms.flagms("--above %g"%t0,"-f threshold -c");
    if t1:
      ms.flagms("--fm-above %g"%t1,"-f fmthreshold -c");

  # plot residuals
  if plotvis:
    try:
      info("Plotting visibilities ($plotvis)");
      ms.PLOTVIS = plotvis;
      ms.plotms("-o ${OUTFILE}_${output}${_s<STEP}${_<label}.png");
    except:
      traceback.print_exc();
      plotfail("plot routine failed, see exception above");                  
    
  # make images
  im.make_image(msname,column=STEFCAL_OUTPUT_COLUMN,dirty=dirty,restore=restore,restore_lsm=restore_lsm);