def sim_cal():
    """ Simulate and Calibrate """
   
    # Make empty MS
    create_empty_ms(synthesis=SYNTHESIS)
    # Estimate image noise
    sigma_vis = compute_vis_noise(SEFD)
    sigma_im = image_noise(sigma_vis)
    
    # Create random Sky
    sky_model(spectral=False, fluxrange="%.6f:%.6f"%(10*sigma_im,1)) # fluxes between 2*noise to 10 Jy
    # remove sources with flux 0 in the initial sky model
    remove_sources(lsmname=LSM)

    # Simulalte sky into MS
    simsky(column='DATA', sefd=420)
    # calibrate 
    calibrate(output='CORR_DATA')
    opts = {}
    if im.IMAGER == "wsclean":
        opts["mgain"] = 0.1
    # imaging
    im.make_image(dirty=True, psf=True, residual=False, restore=True, column='CORRECTED_DATA',
                  restore_lsm=False, **opts)
    
     #Run sourcefinder and make plots
    find_sources(reliability=False)
 
    # calculates intrinsic fluxes using primary beam gain
    #apply_primarybeam()
    
    # crosss match the sources
    matching()
def cal_ms(lsm0='$LSM0', start=0, stop=4):
    """ Run pipeline on a single MS"""
   
    lsm0 = II(lsm0)
   
    # Calibrate each MS
    #run_cmd = lambda : calibrate(lsmname=lsm0)
    #run_cmd = calibrate(lsmname=lsm0)
    v.MS = CONCAT_MS
    calibrate(lsmname=lsm0)
    
    #pper("MS",run_cmd)
    #calibrate(lsmname=lsm0)
    # image combined MS
    #ms.virtconcat(output=CONCAT_MS)
    
    v.MS = CONCAT_MS
    im.make_image(restore=True, psf=True, restore_lsm=False)
	
    # run source finder
    lsm.pybdsm_search(thresh_pix=5 , thresh_isl=3)
    v.LSM = lsm.PYBDSM_OUTPUT
    x.sh("tigger-convert --append $LSM $LSM0 $LSMFINAL -f")

    # make final restored map
    x.sh("tigger-restore ${im.RESTORED_IMAGE} $LSM0 ${im.FULLREST_IMAGE} -f ")
def cal_DI(msname="$MS",
           lsm0='$LSM0',
           timeint=None,
           freqint=None,
           smooth=None,
           start=1,
           stop=5,
           **kw):
    """ Run pipeline on a single MS"""

    msname, lsm0 = interpolate_locals("msname lsm0")

    # Use the initial model to do the direction independent calibration.

    calibrate_DI(msname, lsm0)
    im.make_image(restore=True, psf=True, restore_lsm=False)

    lsm.pybdsm_search(thresh_pix=5, thresh_isl=3)
    v.LSM = lsm.PYBDSM_OUTPUT
    lsm1 = v.LSM

    x.sh("tigger-convert --append $LSM $lsm0 $LSMFINAL -f")

    x.sh(
        "tigger-restore ${im.RESTORED_IMAGE} $LSMFINAL ${im.FULLREST_IMAGE} -f "
    )
示例#4
0
def cal_ms(lsm0='$LSM0', start=0, stop=4):
    """ Run pipeline on a single MS"""

    lsm0 = II(lsm0)

    # Calibrate each MS
    run_cmd = lambda: calibrate(lsmname=lsm0)
    pper("MS", run_cmd)

    # image combined MS
    ms.virtconcat(output=CONCAT_MS)
    v.MS = CONCAT_MS
    im.make_image(restore=True, psf=True, restore_lsm=False)

    # run source finder
    lsm.pybdsm_search(thresh_pix=5, thresh_isl=3)
    v.LSM = lsm.PYBDSM_OUTPUT
    x.sh("tigger-convert --append $LSM $LSM0 $LSMFINAL -f")

    # make final restored map
    x.sh("tigger-restore ${im.RESTORED_IMAGE} $LSM0 ${im.FULLREST_IMAGE} -f ")
def sim_cal():
    """ Simulate and Calibrate """

    # Make empty MS
    create_empty_ms(synthesis=SYNTHESIS)
    # Estimate image noise
    sigma_vis = compute_vis_noise(SEFD)
    sigma_im = image_noise(sigma_vis)

    # Create random Sky
    sky_model(spectral=False, fluxrange="%.6f:%.6f" %
              (10 * sigma_im, 1))  # fluxes between 2*noise to 10 Jy
    # remove sources with flux 0 in the initial sky model
    remove_sources(lsmname=LSM)

    # Simulalte sky into MS
    simsky(column='DATA', sefd=420)
    # calibrate
    calibrate(output='CORR_DATA')
    opts = {}
    if im.IMAGER == "wsclean":
        opts["mgain"] = 0.1
    # imaging
    im.make_image(dirty=True,
                  psf=True,
                  residual=False,
                  restore=True,
                  column='CORRECTED_DATA',
                  restore_lsm=False,
                  **opts)

    #Run sourcefinder and make plots
    find_sources(reliability=False)

    # calculates intrinsic fluxes using primary beam gain
    #apply_primarybeam()

    # crosss match the sources
    matching()
def run_all():

    info(
        "############## Make image from the data column, Model the sky and dE tag ######################"
    )
    im.make_image(column="DATA", restore=True, psf=True, restore_lsm=False)

    make_clean_model()
    info("######################## Calibrating G+B only ####################")

    cal_DI(lsm0=LSM0)

    im.make_image(column="CORRECTED_DATA",
                  restore=True,
                  psf=True,
                  restore_lsm=False)
    image = im.RESTORED_IMAGE

    lsm.pybdsm_search(image, thresh_pix=5, thresh_isl=3)
    v.LSM = lsm.PYBDSM_OUTPUT
    lsm1 = v.LSM

    make_clean_model(image=im.RESTORED_IMAGE, lsm0=LSM, threshold=3)

    info(
        "################# CALIBRATING DIFFENTIAL GAIN (dE): the flyswatter (O.M.Smirnov) ###################"
    )
    cal_DD(lsm0=LSMFINAL)
    im.make_image(column="CORRECTED_DATA",
                  restore=True,
                  psf=True,
                  restore_lsm=False)
    image = im.RESTORED_IMAGE
    lsm.pybdsm_search(image, thresh_pix=5, thresh_isl=3)
    v.LSM = lsm.PYBDSM_OUTPUT
    lsm1 = v.LSM
    make_clean_model(image=im.RESTORED_IMAGE, lsm0=LSM, threshold=3)

    info(
        "############## Change the apparent flux to intrinsic flux for beam calibration ######################"
    )

    APP_INT(lsmname=LSM0, output=LSM_INTRINSIC)

    info(
        "############## Do only G+B with beam and with beam ######################"
    )
    cal_BM(lsm0=LSM_INTRINSIC)

    x.sh("tigger-restore ${im.RESTORED_IMAGE} $LSM0 ${im.FULLREST_IMAGE} -f ")
示例#7
0
def azishe(config='$CFG'):

    # Get parameters from json file
    with open(II(config)) as jsn_std:
        jparams = json.load(jsn_std)

    # Remove empty strings and convert unicode characters to strings
    params = {}
    for key, val in jparams.iteritems():
        # Make sure all keys are strings
        _key = str(key)

        # ignore empty strings and comments
        if val == "" or _key == "#":
            pass
        # convert unicode values to strings
        elif isinstance(val, unicode):
            params[_key] = str(val).lower()
        else:
            params[_key] = val

    get_opts = lambda prefix: filter(lambda a: a[0].startswith(prefix),
                                     params.items())

    # Retrieve MS and imager options
    ms_dict = dict([(key.split('ms_')[-1], val)
                    for (key, val) in get_opts('ms_')])
    im_dict = dict([(key.split('im_')[-1], val)
                    for (key, val) in get_opts('im_')])

    # Seperate deconvolution settings
    _deconv = {}
    for dcv in 'lwimager wsclean moresane casa'.split():
        if params[dcv]:
            _deconv.update({
                dcv:
                dict([(key.split(dcv + '_')[-1], val)
                      for (key, val) in get_opts(dcv + '_')])
            })

    # Set imaging options
    im.IMAGER = imager.IMAGER = params['imager'].lower()
    for opt in 'npix weight robust mode stokes'.split():
        if opt in im_dict.keys():
            setattr(im, opt, im_dict.pop(opt))
    im.cellsize = '%farcsec' % (im_dict.pop('cellsize'))
    im.stokes = im.stokes.upper()

    weight_fov = im_dict.pop('weight_fov')
    if weight_fov:
        im.weight_fov = '%farcmin' % weight_fov

    # Create empty MS
    synthesis = ms_dict.pop('synthesis')
    scalenoise = 1
    if synthesis > 12:
        scalenoise = math.sqrt(12.0 / synthesis)

    msname = II('rodrigues%f.MS' % (time.time()))
    obs = params['observatory'].lower()
    antennas = _OBS[obs]

    freq0 = ms_dict.pop('freq0') * 1e6
    dfreq = ms_dict.pop('dfreq') * 1e3
    direction = "J2000,%s,%s" % (ms_dict.pop('ra'), ms_dict.pop('dec'))
    ms.create_empty_ms(msname=msname,
                       synthesis=synthesis,
                       freq0=freq0,
                       dfreq=dfreq,
                       tel=obs,
                       pos='%s/%s' % (OBSDIR, antennas),
                       direction=direction,
                       **ms_dict)
    if exists(msname):
        v.MS = msname
    else:
        abort(
            "Something went wrong while creating the empty MS. Please check logs for details"
        )

    makedir(DESTDIR)
    ms.plot_uvcov(ms=.1,
                  width=10,
                  height=10,
                  dpi=150,
                  save="$OUTFILE-uvcov.png")

    # Set noise for simulation
    spwtab = ms.ms(subtable="SPECTRAL_WINDOW")
    freq0 = spwtab.getcol("CHAN_FREQ")[ms.SPWID, 0]
    spwtab.close()
    if params['add_noise']:
        noise = compute_vis_noise(sefd=get_sefd(freq0)) * scalenoise
    else:
        noise = 0

    # Simulate Visibilities
    _katalog = params['katalog_id']
    if _katalog:
        katalog = '%s/%s' % (KATDIR, _KATALOG[params['katalog_id']])

        lsmname = II('${OUTDIR>/}${MS:BASE}.lsm.html')

        radius = float(params['radius'])
        fluxrange = params['fluxrange'].split('-')
        if len(fluxrange) > 1:
            fluxrange = map(float, fluxrange)
        elif len(fluxrange) == 1:
            fluxrange = [0, float(fluxrange[0])]

        select = ''
        fits = verify_sky(katalog) == 'FITS'
        if radius or fluxrange:
            if radius: select += '--select="r<%fdeg" ' % radius
            if fluxrange:
                select += '--select="I<%f" ' % fluxrange[1]
                select += '--select="I>%f" ' % fluxrange[0]
        if not fits:
            x.sh(
                'tigger-convert $select --recenter=$direction $katalog $lsmname -f'
            )
        else:
            from pyrap.measures import measures
            dm = measures()
            direction = dm.direction('J2000', ms_opts[ra], ms_opts[dec])
            ra = np.rad2deg(direction['m0']['value'])
            dec = np.rad2deg(direction['m1']['value'])
            hdu = pyfits.open(temp_file)
            hdu[0].hdr['CRVAL1'] = ra
            hdu[0].hdr['CRVAL2'] = dec
            hdu.writeto(tmp_file, clobber=True)

        simsky(lsmname=lsmname,
               tdlsec='turbo-sim:custom',
               noise=noise,
               column='CORRECTED_DATA')

    skymodel = params['sky_model']
    if skymodel:
        simsky(lsmname=skymodel,
               noise=0 if katalog else noise,
               addToCol='CORRECTED_DATA')

    ## Finally Lets image
    # make dirty map
    im.make_image(psf=True, **im_dict)

    # Deconvolve
    for deconv in _deconv:
        deconv = deconv.lower()
        if deconv in STAND_ALONE_DECONV:
            im.make_image(algorithm=deconv,
                          restore=_deconv[deconv],
                          restore_lsm=False,
                          **im_dict)
        else:
            im.IMAGER = deconv
            im.make_image(dirty=False,
                          restore=_deconv[deconv],
                          restore_lsm=False,
                          **im_dict)

    xo.sh('tar -czvf ${OUTDIR>/}${MS:BASE}.tar.gz $msname')
示例#8
0
def stefcal(msname="$MS",
            section="$STEFCAL_SECTION",
            diffgains=None,
            apply_only=False,
            reset=False,
            gain_apply_only=False,
            gain_reset=False,
            diffgain_apply_only=False,
            diffgain_reset=False,
            gain_plot_prefix="$STEFCAL_GAIN_PLOT_PREFIX",
            gain1_plot_prefix="$STEFCAL_GAIN1_PLOT_PREFIX",
            ifrgain_plot_prefix="$STEFCAL_IFRGAIN_PLOT_PREFIX",
            diffgain_plot_prefix="$STEFCAL_DIFFGAIN_PLOT_PREFIX",
            ifrgain_apply_only=False,
            ifrgain_reset=False,
            gain_intervals=None,
            gain_smoothing=None,
            diffgain_intervals=None,
            diffgain_smoothing=None,
            flag_threshold=None,
            calibrate_ifrs="$STEFCAL_CALIBRATE_IFRS",
            input_column="$STEFCAL_INPUT_COLUMN",
            output_column="$STEFCAL_OUTPUT_COLUMN",
            output="CORR_RES",
            plotvis="${ms.PLOTVIS}",
            dirty=True,
            restore=False,
            restore_lsm=True,
            label=None,
            saveconfig="$STEFCAL_SAVE_CONFIG",
            plotfail=None,
            args=[],
            options={},
            **kws):
    """Generic function to run a stefcal job.
  
  'section'         TDL config file section
  'label'           will be assigned to the global LABEL for purposes of file naming
  'apply_only'      if true, will only apply saved solutions rather than re-solve
  '{gain,diffgain,ifrgain}_apply_only'      
                    if true, will only apply saved gain/diffgain/IFR gain solutions rather than re-solve
  'reset'           if true, will reset all saved solutuions prior to starting
  '{gain,diffgain,ifrgain}_reset'      
                    if true, will reset saved gain/diffgain/IFR gain solutuions prior to starting
  'diffgains'       set to a source subset string to solve for diffgains. Set to True to use "=dE"
  'diffgain_mode'   'solve-save' to solve & save, 'solve-nosave' to not save, 'apply' to apply only
  'diffgain_plot'   automatically invoke make_diffgain_plots() if True
  'flag_threshold'  threshold flaging post-solutions. Give one threshold to flag with --above,
                    or T1,T2 for --above T1 --fm-above T2
  'output'          output visibilities ('CORR_DATA','CORR_RES', 'RES' are useful)
  'plotvis'         if not empty, specifies which output visibilities to plot using plot-ms (see plot.ms.py --help) 
  'dirty','restore' 
  'restore_lsm'     image output visibilities (passed to imager.make_image above as is)
  'plotfail'        plotting failure reported via warn or abort. Default is warn, set to abort to abort. 
  'saveconfig'      saves the effective TDL config to file[:section]
  'args','options'  passed to the stefcal job as is (as a list of arguments and kw=value pairs), 
                    can be used to supply extra TDL options
  extra keywords:   passed to the stefcal job as kw=value, can be used to supply extra TDL options, thus
                    overriding settings in the TDL config file. Useful arguments of this kind are e.g.:
                    stefcal_reset_all=True to remove prior gains solutions.
  """
    msname,section,lsm,label,plotvis,calibrate_ifrs, \
      gain_plot_prefix,gain1_plot_prefix,ifrgain_plot_prefix,diffgain_plot_prefix,saveconfig,plotfail,input_column,output_column = \
      interpolate_locals("msname section lsm label plotvis calibrate_ifrs gain_plot_prefix gain1_plot_prefix ifrgain_plot_prefix diffgain_plot_prefix "
        "saveconfig plotfail input_column output_column")

    plotfail = plotfail or warn
    makedir(v.DESTDIR)

    # increment step counter and assign global label

    if label is not None:
        v.LABEL = str(label)
    if type(v.STEP) is int and STEFCAL_STEP_INCR:
        v.STEP += STEFCAL_STEP_INCR

    # setup stefcal options and run
    info("Running stefcal ${step <STEP} ${(<LABEL>)}")
    # setup args
    args0 = [
        """${ms.MS_TDL} ${ms.CHAN_TDL} ${lsm.LSM_TDL} ms_sel.ms_ifr_subset_str=${ms.IFRS} 
    ms_sel.input_column=$input_column
    ms_sel.output_column=$output_column
    stefcal_gain.enabled=1 stefcal_diffgain.enabled=%d %s""" %
        ((1 if diffgains else 0), STEFCAL_TDLOPTS)
    ]
    if diffgains:
        if diffgains is True:
            diffgains = "=dE"
        args0.append(
            "de_subset.subset_enabled=1 de_subset.source_subset=$diffgains")
    opts = {
        'do_output':
        output,
        'calibrate_ifrs':
        calibrate_ifrs,
        'stefcal_gain.mode':
        "apply" if apply_only or gain_apply_only else "solve-save",
        'stefcal_gain1.mode':
        "apply" if apply_only or gain_apply_only else "solve-save",
        'stefcal_gain.reset':
        int(reset or gain_reset),
        'stefcal_diffgain.mode':
        "apply" if apply_only or diffgain_apply_only else "solve-save",
        'stefcal_diffgain1.mode':
        "apply" if apply_only or diffgain_apply_only else "solve-save",
        'stefcal_diffgain.reset':
        int(reset or diffgain_reset),
        'stefcal_diffgain1.reset':
        int(reset or diffgain_reset),
        'stefcal_ifr_gain_mode':
        "apply" if apply_only or ifrgain_apply_only else "solve-save",
        'stefcal_ifr_gain_reset':
        int(reset or ifrgain_reset),
        'stefcal_gain.table':
        STEFCAL_GAIN,
        'stefcal_gain1.table':
        STEFCAL_GAIN1,
        'stefcal_diffgain.table':
        STEFCAL_DIFFGAIN,
        'stefcal_diffgain1.table':
        STEFCAL_DIFFGAIN1,
        'stefcal_ifr_gain_table':
        STEFCAL_IFRGAIN,
        'stefcal_visualize':
        False
    }
    # set gain parameters
    if gain_smoothing or STEFCAL_GAIN_SMOOTHING or gain_intervals or STEFCAL_GAIN_INTERVALS:
        timesmooth, freqsmooth = gain_smoothing or STEFCAL_GAIN_SMOOTHING or (
            0, 0)
        timeint, freqint = gain_intervals or STEFCAL_GAIN_INTERVALS or (0, 0)
        opts['stefcal_gain.timeint'] = 0 if timesmooth else timeint
        opts['stefcal_gain.freqint'] = 0 if freqsmooth else freqint
        opts['stefcal_gain.timesmooth'] = timesmooth
        opts['stefcal_gain.freqsmooth'] = freqsmooth
    # set diffgain parameters
    if diffgain_smoothing or STEFCAL_DIFFGAIN_SMOOTHING or diffgain_intervals or STEFCAL_DIFFGAIN_INTERVALS:
        timesmooth, freqsmooth = diffgain_smoothing or STEFCAL_DIFFGAIN_SMOOTHING or (
            0, 0)
        timeint, freqint = diffgain_intervals or STEFCAL_DIFFGAIN_INTERVALS or (
            0, 0)
        opts['stefcal_diffgain.timeint'] = 0 if timesmooth else timeint
        opts['stefcal_diffgain.freqint'] = 0 if freqsmooth else freqint
        opts['stefcal_diffgain.timesmooth'] = timesmooth
        opts['stefcal_diffgain.freqsmooth'] = freqsmooth

    # add user-defined args
    args0 += list(args)
    opts.update(options)
    opts.update(kws)
    # run the job
    mqt.run(STEFCAL_SCRIPT,
            STEFCAL_JOBNAME,
            section=section,
            saveconfig=saveconfig,
            args=args0,
            options=opts)

    # copy gains
    try:
        if not apply_only:
            if os.path.exists(STEFCAL_GAIN) and not gain_apply_only:
                std.copy(STEFCAL_GAIN, STEFCAL_GAIN_SAVE)
                if gain_plot_prefix:
                    make_gain_plots(STEFCAL_GAIN_SAVE, prefix=gain_plot_prefix)
            if os.path.exists(STEFCAL_GAIN1) and not gain_apply_only:
                std.copy(STEFCAL_GAIN1, STEFCAL_GAIN1_SAVE)
                if gain_plot_prefix:
                    make_gain_plots(STEFCAL_GAIN1_SAVE,
                                    prefix=gain_plot_prefix)
            if os.path.exists(STEFCAL_DIFFGAIN) and not diffgain_apply_only:
                std.copy(STEFCAL_DIFFGAIN, STEFCAL_DIFFGAIN_SAVE)
                if diffgain_plot_prefix:
                    make_diffgain_plots(STEFCAL_DIFFGAIN_SAVE,
                                        prefix=diffgain_plot_prefix)
            if os.path.exists(STEFCAL_IFRGAIN) and not ifrgain_apply_only:
                std.copy(STEFCAL_IFRGAIN, STEFCAL_IFRGAIN_SAVE)
                if ifrgain_plot_prefix:
                    make_ifrgain_plots(STEFCAL_IFRGAIN_SAVE,
                                       prefix=ifrgain_plot_prefix)
    except:
        traceback.print_exc()
        plotfail("plot routine failed, see exception above")
    # post-calibration flagging
    if flag_threshold:
        if isinstance(flag_threshold, (list, tuple)):
            t0, t1 = flag_threshold
        else:
            t0, t1 = flag_threshold, None
        ms.flagms("--above %g" % t0, "-f threshold -c")
        if t1:
            ms.flagms("--fm-above %g" % t1, "-f fmthreshold -c")

    # plot residuals
    if plotvis:
        try:
            info("Plotting visibilities ($plotvis)")
            ms.PLOTVIS = plotvis
            ms.plotms("-o ${OUTFILE}_${output}${_s<STEP}${_<label}.png")
        except:
            traceback.print_exc()
            plotfail("plot routine failed, see exception above")

    # make images
    im.make_image(msname,
                  column=STEFCAL_OUTPUT_COLUMN,
                  dirty=dirty,
                  restore=restore,
                  restore_lsm=restore_lsm)
示例#9
0
def azishe(config='$CFG'):

    # Get parameters from json file    
    with open(II(config)) as jsn_std:
        jparams = json.load(jsn_std)

    # Remove empty strings and convert unicode characters to strings
    params = {}
    for key, val in jparams.iteritems():
        # Make sure all keys are strings
        _key = str(key)

        # ignore empty strings and comments
        if val=="" or _key=="#":
            pass
        # convert unicode values to strings
        elif isinstance(val,unicode):
            params[_key] = str(val).lower()
        else:
            params[_key] = val
 
    get_opts = lambda prefix: filter(lambda a: a[0].startswith(prefix), params.items())

    # Retrieve MS and imager options
    ms_dict = dict([(key.split('ms_')[-1],val) for (key,val) in get_opts('ms_') ] )
    im_dict = dict([(key.split('im_')[-1],val) for (key,val) in get_opts('im_') ] )

    # Seperate deconvolution settings
    _deconv = {}
    for dcv in 'lwimager wsclean moresane casa'.split():
        if params[dcv]:
            _deconv.update( {dcv:dict([(key.split(dcv+'_')[-1],val) for (key,val) in get_opts(dcv+'_') ] )} )
    
    # Set imaging options
    im.IMAGER = imager.IMAGER = params['imager'].lower()
    for opt in 'npix weight robust mode stokes'.split():
        if opt in im_dict.keys():
            setattr(im,opt,im_dict.pop(opt))
    im.cellsize = '%farcsec'%(im_dict.pop('cellsize'))
    im.stokes = im.stokes.upper()

    weight_fov = im_dict.pop('weight_fov')
    if weight_fov:
        im.weight_fov = '%farcmin'%weight_fov
    
    # Create empty MS
    synthesis = ms_dict.pop('synthesis')
    scalenoise = 1
    if synthesis > 12:
        scalenoise = math.sqrt(12.0/synthesis)
    
    msname = II('rodrigues%f.MS'%(time.time())) 
    obs = params['observatory'].lower()
    antennas = _OBS[obs]
    
    freq0 = ms_dict.pop('freq0')*1e6
    dfreq = ms_dict.pop('dfreq')*1e3
    direction = "J2000,%s,%s"%(ms_dict.pop('ra'),ms_dict.pop('dec'))
    ms.create_empty_ms(msname=msname,synthesis=synthesis,freq0=freq0,
                dfreq=dfreq,tel=obs,pos='%s/%s'%(OBSDIR,antennas),
                direction=direction,**ms_dict)
    if exists(msname):
        v.MS = msname
    else:
        abort("Something went wrong while creating the empty MS. Please check logs for details")

    makedir(DESTDIR)
    ms.plot_uvcov(ms=.1,width=10,height=10,dpi=150,save="$OUTFILE-uvcov.png")

    # Set noise for simulation
    spwtab = ms.ms(subtable="SPECTRAL_WINDOW")
    freq0 = spwtab.getcol("CHAN_FREQ")[ms.SPWID,0]
    spwtab.close()
    if params['add_noise']:
        noise =  compute_vis_noise(sefd=get_sefd(freq0)) * scalenoise
    else:
        noise = 0

    # Simulate Visibilities
    _katalog = params['katalog_id']
    if _katalog:
        katalog = '%s/%s'%(KATDIR,_KATALOG[params['katalog_id']])

        lsmname = II('${OUTDIR>/}${MS:BASE}.lsm.html')

        radius = float(params['radius'])
        fluxrange = params['fluxrange'].split('-')
        if len(fluxrange)>1:
            fluxrange = map(float,fluxrange)
        elif len(fluxrange)==1:
            fluxrange = [0,float(fluxrange[0])]

        
        select = ''
        fits = verify_sky(katalog) == 'FITS'
        if radius or fluxrange:
            if radius: select += '--select="r<%fdeg" '%radius
            if fluxrange: 
                select += '--select="I<%f" '%fluxrange[1]
                select += '--select="I>%f" '%fluxrange[0]
        if not fits:
            x.sh('tigger-convert $select --recenter=$direction $katalog $lsmname -f')
        else:
            from pyrap.measures import measures
            dm = measures()
            direction = dm.direction('J2000',ms_opts[ra],ms_opts[dec])
            ra = np.rad2deg(direction['m0']['value'])
            dec = np.rad2deg(direction['m1']['value'])
            hdu = pyfits.open(temp_file)
            hdu[0].hdr['CRVAL1'] = ra
            hdu[0].hdr['CRVAL2'] = dec
            hdu.writeto(tmp_file,clobber=True)

        simsky(lsmname=lsmname,tdlsec='turbo-sim:custom',noise=noise,column='CORRECTED_DATA')

    skymodel = params['sky_model']
    if skymodel:
        simsky(lsmname=skymodel,noise=0 if katalog else noise,addToCol='CORRECTED_DATA')

    ## Finally Lets image
    # make dirty map
    im.make_image(psf=True,**im_dict)

    # Deconvolve
    for deconv in _deconv:
        deconv = deconv.lower()
        if deconv in STAND_ALONE_DECONV:
            im.make_image(algorithm=deconv,restore=_deconv[deconv],restore_lsm=False,**im_dict)
        else:
            im.IMAGER = deconv
            im.make_image(dirty=False,restore=_deconv[deconv],restore_lsm=False,**im_dict)

    xo.sh('tar -czvf ${OUTDIR>/}${MS:BASE}.tar.gz $msname')
示例#10
0
def stefcal ( msname="$MS",section="$STEFCAL_SECTION",
              diffgains=None,
              apply_only=False,
              reset=False,
              gain_apply_only=False,
              gain_reset=False,
              diffgain_apply_only=False,
              diffgain_reset=False,
              gain_plot_prefix="$STEFCAL_GAIN_PLOT_PREFIX",
              gain1_plot_prefix="$STEFCAL_GAIN1_PLOT_PREFIX",
              ifrgain_plot_prefix="$STEFCAL_IFRGAIN_PLOT_PREFIX",
              diffgain_plot_prefix="$STEFCAL_DIFFGAIN_PLOT_PREFIX",
              ifrgain_apply_only=False,
              ifrgain_reset=False,
              gain_intervals=None,gain_smoothing=None,
              diffgain_intervals=None,diffgain_smoothing=None,
              flag_threshold=None,
              calibrate_ifrs="$STEFCAL_CALIBRATE_IFRS",
              input_column="$STEFCAL_INPUT_COLUMN",
              output_column="$STEFCAL_OUTPUT_COLUMN",
              output="CORR_RES",
              plotvis="${ms.PLOTVIS}",
              dirty=True,restore=False,restore_lsm=True,
              label=None,
              saveconfig="$STEFCAL_SAVE_CONFIG",
              plotfail=None,
              args=[],options={},
              **kws):
  """Generic function to run a stefcal job.
  
  'section'         TDL config file section
  'label'           will be assigned to the global LABEL for purposes of file naming
  'apply_only'      if true, will only apply saved solutions rather than re-solve
  '{gain,diffgain,ifrgain}_apply_only'      
                    if true, will only apply saved gain/diffgain/IFR gain solutions rather than re-solve
  'reset'           if true, will reset all saved solutuions prior to starting
  '{gain,diffgain,ifrgain}_reset'      
                    if true, will reset saved gain/diffgain/IFR gain solutuions prior to starting
  'diffgains'       set to a source subset string to solve for diffgains. Set to True to use "=dE"
  'diffgain_mode'   'solve-save' to solve & save, 'solve-nosave' to not save, 'apply' to apply only
  'diffgain_plot'   automatically invoke make_diffgain_plots() if True
  'flag_threshold'  threshold flaging post-solutions. Give one threshold to flag with --above,
                    or T1,T2 for --above T1 --fm-above T2
  'output'          output visibilities ('CORR_DATA','CORR_RES', 'RES' are useful)
  'plotvis'         if not empty, specifies which output visibilities to plot using plot-ms (see plot.ms.py --help) 
  'dirty','restore' 
  'restore_lsm'     image output visibilities (passed to imager.make_image above as is)
  'plotfail'        plotting failure reported via warn or abort. Default is warn, set to abort to abort. 
  'saveconfig'      saves the effective TDL config to file[:section]
  'args','options'  passed to the stefcal job as is (as a list of arguments and kw=value pairs), 
                    can be used to supply extra TDL options
  extra keywords:   passed to the stefcal job as kw=value, can be used to supply extra TDL options, thus
                    overriding settings in the TDL config file. Useful arguments of this kind are e.g.:
                    stefcal_reset_all=True to remove prior gains solutions.
  """
  msname,section,lsm,label,plotvis,calibrate_ifrs, \
    gain_plot_prefix,gain1_plot_prefix,ifrgain_plot_prefix,diffgain_plot_prefix,saveconfig,plotfail,input_column,output_column = \
    interpolate_locals("msname section lsm label plotvis calibrate_ifrs gain_plot_prefix gain1_plot_prefix ifrgain_plot_prefix diffgain_plot_prefix "
      "saveconfig plotfail input_column output_column")
  
  plotfail = plotfail or warn
  makedir(v.DESTDIR);
  
  # increment step counter and assign global label
  
  if label is not None:
    v.LABEL = str(label);
  if type(v.STEP) is int and STEFCAL_STEP_INCR:
    v.STEP += STEFCAL_STEP_INCR;

  # setup stefcal options and run 
  info("Running stefcal ${step <STEP} ${(<LABEL>)}");
  # setup args
  args0 = [ """${ms.MS_TDL} ${ms.CHAN_TDL} ${lsm.LSM_TDL} ms_sel.ms_ifr_subset_str=${ms.IFRS} 
    ms_sel.input_column=$input_column
    ms_sel.output_column=$output_column
    stefcal_gain.enabled=1 stefcal_diffgain.enabled=%d %s"""%
    ((1 if diffgains else 0),STEFCAL_TDLOPTS) ];
  if diffgains:
    if diffgains is True:
      diffgains = "=dE";
    args0.append("de_subset.subset_enabled=1 de_subset.source_subset=$diffgains"); 
  opts = {
    'do_output': output,
    'calibrate_ifrs': calibrate_ifrs,
    'stefcal_gain.mode': "apply" if apply_only or gain_apply_only else "solve-save",
    'stefcal_gain1.mode': "apply" if apply_only  or gain_apply_only else "solve-save",
    'stefcal_gain.reset': int(reset or gain_reset),
    'stefcal_diffgain.mode': "apply" if apply_only or diffgain_apply_only else "solve-save",
    'stefcal_diffgain1.mode': "apply" if apply_only or diffgain_apply_only  else "solve-save",
    'stefcal_diffgain.reset': int(reset or diffgain_reset),
    'stefcal_diffgain1.reset': int(reset or diffgain_reset),
    'stefcal_ifr_gain_mode': "apply" if apply_only or ifrgain_apply_only else "solve-save",
    'stefcal_ifr_gain_reset': int(reset or ifrgain_reset),
    'stefcal_gain.table': STEFCAL_GAIN,
    'stefcal_gain1.table': STEFCAL_GAIN1,
    'stefcal_diffgain.table': STEFCAL_DIFFGAIN,
    'stefcal_diffgain1.table': STEFCAL_DIFFGAIN1,
    'stefcal_ifr_gain_table': STEFCAL_IFRGAIN,
    'stefcal_visualize': False
  }
  # set gain parameters
  if gain_smoothing or STEFCAL_GAIN_SMOOTHING or gain_intervals or STEFCAL_GAIN_INTERVALS:
      timesmooth,freqsmooth = gain_smoothing or STEFCAL_GAIN_SMOOTHING or (0,0);
      timeint,freqint = gain_intervals or STEFCAL_GAIN_INTERVALS or (0,0);
      opts['stefcal_gain.timeint'] = 0 if timesmooth else timeint;
      opts['stefcal_gain.freqint'] = 0 if freqsmooth else freqint;
      opts['stefcal_gain.timesmooth'] = timesmooth;
      opts['stefcal_gain.freqsmooth'] = freqsmooth;
  # set diffgain parameters
  if diffgain_smoothing or STEFCAL_DIFFGAIN_SMOOTHING or diffgain_intervals or STEFCAL_DIFFGAIN_INTERVALS:
      timesmooth,freqsmooth = diffgain_smoothing or STEFCAL_DIFFGAIN_SMOOTHING or (0,0);
      timeint,freqint = diffgain_intervals or STEFCAL_DIFFGAIN_INTERVALS or (0,0);
      opts['stefcal_diffgain.timeint'] = 0 if timesmooth else timeint;
      opts['stefcal_diffgain.freqint'] = 0 if freqsmooth else freqint;
      opts['stefcal_diffgain.timesmooth'] = timesmooth;
      opts['stefcal_diffgain.freqsmooth'] = freqsmooth;

  # add user-defined args
  args0 += list(args);
  opts.update(options);
  opts.update(kws);
  # run the job
  mqt.run(STEFCAL_SCRIPT,STEFCAL_JOBNAME,section=section,saveconfig=saveconfig,args=args0,options=opts);
  
  # copy gains
  try:
    if not apply_only:
      if os.path.exists(STEFCAL_GAIN) and not gain_apply_only:
        std.copy(STEFCAL_GAIN,STEFCAL_GAIN_SAVE);
        if gain_plot_prefix:
          make_gain_plots(STEFCAL_GAIN_SAVE,prefix=gain_plot_prefix);
      if os.path.exists(STEFCAL_GAIN1) and not gain_apply_only:
        std.copy(STEFCAL_GAIN1,STEFCAL_GAIN1_SAVE);
        if gain_plot_prefix:
          make_gain_plots(STEFCAL_GAIN1_SAVE,prefix=gain_plot_prefix);
      if os.path.exists(STEFCAL_DIFFGAIN) and not diffgain_apply_only:
        std.copy(STEFCAL_DIFFGAIN,STEFCAL_DIFFGAIN_SAVE);
        if diffgain_plot_prefix:
          make_diffgain_plots(STEFCAL_DIFFGAIN_SAVE,prefix=diffgain_plot_prefix);
      if os.path.exists(STEFCAL_IFRGAIN) and not ifrgain_apply_only:
        std.copy(STEFCAL_IFRGAIN,STEFCAL_IFRGAIN_SAVE);
        if ifrgain_plot_prefix:
          make_ifrgain_plots(STEFCAL_IFRGAIN_SAVE,prefix=ifrgain_plot_prefix);
  except:
    traceback.print_exc();
    plotfail("plot routine failed, see exception above");                  
  # post-calibration flagging
  if flag_threshold:
    if isinstance(flag_threshold,(list,tuple)):
      t0,t1 = flag_threshold;
    else:
      t0,t1 = flag_threshold,None;
    ms.flagms("--above %g"%t0,"-f threshold -c");
    if t1:
      ms.flagms("--fm-above %g"%t1,"-f fmthreshold -c");

  # plot residuals
  if plotvis:
    try:
      info("Plotting visibilities ($plotvis)");
      ms.PLOTVIS = plotvis;
      ms.plotms("-o ${OUTFILE}_${output}${_s<STEP}${_<label}.png");
    except:
      traceback.print_exc();
      plotfail("plot routine failed, see exception above");                  
    
  # make images
  im.make_image(msname,column=STEFCAL_OUTPUT_COLUMN,dirty=dirty,restore=restore,restore_lsm=restore_lsm);