def sim_cal():
""" Simulate and Calibrate """
# Make empty MS
create_empty_ms(synthesis=SYNTHESIS)
# Estimate image noise
sigma_vis = compute_vis_noise(SEFD)
sigma_im = image_noise(sigma_vis)
# Create random Sky
sky_model(spectral=False, fluxrange="%.6f:%.6f"%(10*sigma_im,1)) # fluxes between 2*noise to 10 Jy
# remove sources with flux 0 in the initial sky model
remove_sources(lsmname=LSM)
# Simulalte sky into MS
simsky(column='DATA', sefd=420)
# calibrate
calibrate(output='CORR_DATA')
opts = {}
if im.IMAGER == "wsclean":
opts["mgain"] = 0.1
# imaging
im.make_image(dirty=True, psf=True, residual=False, restore=True, column='CORRECTED_DATA',
restore_lsm=False, **opts)
#Run sourcefinder and make plots
find_sources(reliability=False)
# calculates intrinsic fluxes using primary beam gain
#apply_primarybeam()
# crosss match the sources
matching()
def cal_ms(lsm0='$LSM0', start=0, stop=4):
""" Run pipeline on a single MS"""
lsm0 = II(lsm0)
# Calibrate each MS
#run_cmd = lambda : calibrate(lsmname=lsm0)
#run_cmd = calibrate(lsmname=lsm0)
v.MS = CONCAT_MS
calibrate(lsmname=lsm0)
#pper("MS",run_cmd)
#calibrate(lsmname=lsm0)
# image combined MS
#ms.virtconcat(output=CONCAT_MS)
v.MS = CONCAT_MS
im.make_image(restore=True, psf=True, restore_lsm=False)
# run source finder
lsm.pybdsm_search(thresh_pix=5 , thresh_isl=3)
v.LSM = lsm.PYBDSM_OUTPUT
x.sh("tigger-convert --append $LSM $LSM0 $LSMFINAL -f")
# make final restored map
x.sh("tigger-restore ${im.RESTORED_IMAGE} $LSM0 ${im.FULLREST_IMAGE} -f ")
def cal_DI(msname="$MS",
lsm0='$LSM0',
timeint=None,
freqint=None,
smooth=None,
start=1,
stop=5,
**kw):
""" Run pipeline on a single MS"""
msname, lsm0 = interpolate_locals("msname lsm0")
# Use the initial model to do the direction independent calibration.
calibrate_DI(msname, lsm0)
im.make_image(restore=True, psf=True, restore_lsm=False)
lsm.pybdsm_search(thresh_pix=5, thresh_isl=3)
v.LSM = lsm.PYBDSM_OUTPUT
lsm1 = v.LSM
x.sh("tigger-convert --append $LSM $lsm0 $LSMFINAL -f")
x.sh(
"tigger-restore ${im.RESTORED_IMAGE} $LSMFINAL ${im.FULLREST_IMAGE} -f "
)
def cal_ms(lsm0='$LSM0', start=0, stop=4):
""" Run pipeline on a single MS"""
lsm0 = II(lsm0)
# Calibrate each MS
run_cmd = lambda: calibrate(lsmname=lsm0)
pper("MS", run_cmd)
# image combined MS
ms.virtconcat(output=CONCAT_MS)
v.MS = CONCAT_MS
im.make_image(restore=True, psf=True, restore_lsm=False)
# run source finder
lsm.pybdsm_search(thresh_pix=5, thresh_isl=3)
v.LSM = lsm.PYBDSM_OUTPUT
x.sh("tigger-convert --append $LSM $LSM0 $LSMFINAL -f")
# make final restored map
x.sh("tigger-restore ${im.RESTORED_IMAGE} $LSM0 ${im.FULLREST_IMAGE} -f ")
def sim_cal():
""" Simulate and Calibrate """
# Make empty MS
create_empty_ms(synthesis=SYNTHESIS)
# Estimate image noise
sigma_vis = compute_vis_noise(SEFD)
sigma_im = image_noise(sigma_vis)
# Create random Sky
sky_model(spectral=False, fluxrange="%.6f:%.6f" %
(10 * sigma_im, 1)) # fluxes between 2*noise to 10 Jy
# remove sources with flux 0 in the initial sky model
remove_sources(lsmname=LSM)
# Simulalte sky into MS
simsky(column='DATA', sefd=420)
# calibrate
calibrate(output='CORR_DATA')
opts = {}
if im.IMAGER == "wsclean":
opts["mgain"] = 0.1
# imaging
im.make_image(dirty=True,
psf=True,
residual=False,
restore=True,
column='CORRECTED_DATA',
restore_lsm=False,
**opts)
#Run sourcefinder and make plots
find_sources(reliability=False)
# calculates intrinsic fluxes using primary beam gain
#apply_primarybeam()
# crosss match the sources
matching()
def run_all():
info(
"############## Make image from the data column, Model the sky and dE tag ######################"
)
im.make_image(column="DATA", restore=True, psf=True, restore_lsm=False)
make_clean_model()
info("######################## Calibrating G+B only ####################")
cal_DI(lsm0=LSM0)
im.make_image(column="CORRECTED_DATA",
restore=True,
psf=True,
restore_lsm=False)
image = im.RESTORED_IMAGE
lsm.pybdsm_search(image, thresh_pix=5, thresh_isl=3)
v.LSM = lsm.PYBDSM_OUTPUT
lsm1 = v.LSM
make_clean_model(image=im.RESTORED_IMAGE, lsm0=LSM, threshold=3)
info(
"################# CALIBRATING DIFFENTIAL GAIN (dE): the flyswatter (O.M.Smirnov) ###################"
)
cal_DD(lsm0=LSMFINAL)
im.make_image(column="CORRECTED_DATA",
restore=True,
psf=True,
restore_lsm=False)
image = im.RESTORED_IMAGE
lsm.pybdsm_search(image, thresh_pix=5, thresh_isl=3)
v.LSM = lsm.PYBDSM_OUTPUT
lsm1 = v.LSM
make_clean_model(image=im.RESTORED_IMAGE, lsm0=LSM, threshold=3)
info(
"############## Change the apparent flux to intrinsic flux for beam calibration ######################"
)
APP_INT(lsmname=LSM0, output=LSM_INTRINSIC)
info(
"############## Do only G+B with beam and with beam ######################"
)
cal_BM(lsm0=LSM_INTRINSIC)
x.sh("tigger-restore ${im.RESTORED_IMAGE} $LSM0 ${im.FULLREST_IMAGE} -f ")
def azishe(config='$CFG'):
# Get parameters from json file
with open(II(config)) as jsn_std:
jparams = json.load(jsn_std)
# Remove empty strings and convert unicode characters to strings
params = {}
for key, val in jparams.iteritems():
# Make sure all keys are strings
_key = str(key)
# ignore empty strings and comments
if val == "" or _key == "#":
pass
# convert unicode values to strings
elif isinstance(val, unicode):
params[_key] = str(val).lower()
else:
params[_key] = val
get_opts = lambda prefix: filter(lambda a: a[0].startswith(prefix),
params.items())
# Retrieve MS and imager options
ms_dict = dict([(key.split('ms_')[-1], val)
for (key, val) in get_opts('ms_')])
im_dict = dict([(key.split('im_')[-1], val)
for (key, val) in get_opts('im_')])
# Seperate deconvolution settings
_deconv = {}
for dcv in 'lwimager wsclean moresane casa'.split():
if params[dcv]:
_deconv.update({
dcv:
dict([(key.split(dcv + '_')[-1], val)
for (key, val) in get_opts(dcv + '_')])
})
# Set imaging options
im.IMAGER = imager.IMAGER = params['imager'].lower()
for opt in 'npix weight robust mode stokes'.split():
if opt in im_dict.keys():
setattr(im, opt, im_dict.pop(opt))
im.cellsize = '%farcsec' % (im_dict.pop('cellsize'))
im.stokes = im.stokes.upper()
weight_fov = im_dict.pop('weight_fov')
if weight_fov:
im.weight_fov = '%farcmin' % weight_fov
# Create empty MS
synthesis = ms_dict.pop('synthesis')
scalenoise = 1
if synthesis > 12:
scalenoise = math.sqrt(12.0 / synthesis)
msname = II('rodrigues%f.MS' % (time.time()))
obs = params['observatory'].lower()
antennas = _OBS[obs]
freq0 = ms_dict.pop('freq0') * 1e6
dfreq = ms_dict.pop('dfreq') * 1e3
direction = "J2000,%s,%s" % (ms_dict.pop('ra'), ms_dict.pop('dec'))
ms.create_empty_ms(msname=msname,
synthesis=synthesis,
freq0=freq0,
dfreq=dfreq,
tel=obs,
pos='%s/%s' % (OBSDIR, antennas),
direction=direction,
**ms_dict)
if exists(msname):
v.MS = msname
else:
abort(
"Something went wrong while creating the empty MS. Please check logs for details"
)
makedir(DESTDIR)
ms.plot_uvcov(ms=.1,
width=10,
height=10,
dpi=150,
save="$OUTFILE-uvcov.png")
# Set noise for simulation
spwtab = ms.ms(subtable="SPECTRAL_WINDOW")
freq0 = spwtab.getcol("CHAN_FREQ")[ms.SPWID, 0]
spwtab.close()
if params['add_noise']:
noise = compute_vis_noise(sefd=get_sefd(freq0)) * scalenoise
else:
noise = 0
# Simulate Visibilities
_katalog = params['katalog_id']
if _katalog:
katalog = '%s/%s' % (KATDIR, _KATALOG[params['katalog_id']])
lsmname = II('${OUTDIR>/}${MS:BASE}.lsm.html')
radius = float(params['radius'])
fluxrange = params['fluxrange'].split('-')
if len(fluxrange) > 1:
fluxrange = map(float, fluxrange)
elif len(fluxrange) == 1:
fluxrange = [0, float(fluxrange[0])]
select = ''
fits = verify_sky(katalog) == 'FITS'
if radius or fluxrange:
if radius: select += '--select="r<%fdeg" ' % radius
if fluxrange:
select += '--select="I<%f" ' % fluxrange[1]
select += '--select="I>%f" ' % fluxrange[0]
if not fits:
x.sh(
'tigger-convert $select --recenter=$direction $katalog $lsmname -f'
)
else:
from pyrap.measures import measures
dm = measures()
direction = dm.direction('J2000', ms_opts[ra], ms_opts[dec])
ra = np.rad2deg(direction['m0']['value'])
dec = np.rad2deg(direction['m1']['value'])
hdu = pyfits.open(temp_file)
hdu[0].hdr['CRVAL1'] = ra
hdu[0].hdr['CRVAL2'] = dec
hdu.writeto(tmp_file, clobber=True)
simsky(lsmname=lsmname,
tdlsec='turbo-sim:custom',
noise=noise,
column='CORRECTED_DATA')
skymodel = params['sky_model']
if skymodel:
simsky(lsmname=skymodel,
noise=0 if katalog else noise,
addToCol='CORRECTED_DATA')
## Finally Lets image
# make dirty map
im.make_image(psf=True, **im_dict)
# Deconvolve
for deconv in _deconv:
deconv = deconv.lower()
if deconv in STAND_ALONE_DECONV:
im.make_image(algorithm=deconv,
restore=_deconv[deconv],
restore_lsm=False,
**im_dict)
else:
im.IMAGER = deconv
im.make_image(dirty=False,
restore=_deconv[deconv],
restore_lsm=False,
**im_dict)
xo.sh('tar -czvf ${OUTDIR>/}${MS:BASE}.tar.gz $msname')
def stefcal(msname="$MS",
section="$STEFCAL_SECTION",
diffgains=None,
apply_only=False,
reset=False,
gain_apply_only=False,
gain_reset=False,
diffgain_apply_only=False,
diffgain_reset=False,
gain_plot_prefix="$STEFCAL_GAIN_PLOT_PREFIX",
gain1_plot_prefix="$STEFCAL_GAIN1_PLOT_PREFIX",
ifrgain_plot_prefix="$STEFCAL_IFRGAIN_PLOT_PREFIX",
diffgain_plot_prefix="$STEFCAL_DIFFGAIN_PLOT_PREFIX",
ifrgain_apply_only=False,
ifrgain_reset=False,
gain_intervals=None,
gain_smoothing=None,
diffgain_intervals=None,
diffgain_smoothing=None,
flag_threshold=None,
calibrate_ifrs="$STEFCAL_CALIBRATE_IFRS",
input_column="$STEFCAL_INPUT_COLUMN",
output_column="$STEFCAL_OUTPUT_COLUMN",
output="CORR_RES",
plotvis="${ms.PLOTVIS}",
dirty=True,
restore=False,
restore_lsm=True,
label=None,
saveconfig="$STEFCAL_SAVE_CONFIG",
plotfail=None,
args=[],
options={},
**kws):
"""Generic function to run a stefcal job.
'section' TDL config file section
'label' will be assigned to the global LABEL for purposes of file naming
'apply_only' if true, will only apply saved solutions rather than re-solve
'{gain,diffgain,ifrgain}_apply_only'
if true, will only apply saved gain/diffgain/IFR gain solutions rather than re-solve
'reset' if true, will reset all saved solutuions prior to starting
'{gain,diffgain,ifrgain}_reset'
if true, will reset saved gain/diffgain/IFR gain solutuions prior to starting
'diffgains' set to a source subset string to solve for diffgains. Set to True to use "=dE"
'diffgain_mode' 'solve-save' to solve & save, 'solve-nosave' to not save, 'apply' to apply only
'diffgain_plot' automatically invoke make_diffgain_plots() if True
'flag_threshold' threshold flaging post-solutions. Give one threshold to flag with --above,
or T1,T2 for --above T1 --fm-above T2
'output' output visibilities ('CORR_DATA','CORR_RES', 'RES' are useful)
'plotvis' if not empty, specifies which output visibilities to plot using plot-ms (see plot.ms.py --help)
'dirty','restore'
'restore_lsm' image output visibilities (passed to imager.make_image above as is)
'plotfail' plotting failure reported via warn or abort. Default is warn, set to abort to abort.
'saveconfig' saves the effective TDL config to file[:section]
'args','options' passed to the stefcal job as is (as a list of arguments and kw=value pairs),
can be used to supply extra TDL options
extra keywords: passed to the stefcal job as kw=value, can be used to supply extra TDL options, thus
overriding settings in the TDL config file. Useful arguments of this kind are e.g.:
stefcal_reset_all=True to remove prior gains solutions.
"""
msname,section,lsm,label,plotvis,calibrate_ifrs, \
gain_plot_prefix,gain1_plot_prefix,ifrgain_plot_prefix,diffgain_plot_prefix,saveconfig,plotfail,input_column,output_column = \
interpolate_locals("msname section lsm label plotvis calibrate_ifrs gain_plot_prefix gain1_plot_prefix ifrgain_plot_prefix diffgain_plot_prefix "
"saveconfig plotfail input_column output_column")
plotfail = plotfail or warn
makedir(v.DESTDIR)
# increment step counter and assign global label
if label is not None:
v.LABEL = str(label)
if type(v.STEP) is int and STEFCAL_STEP_INCR:
v.STEP += STEFCAL_STEP_INCR
# setup stefcal options and run
info("Running stefcal ${step <STEP} ${(<LABEL>)}")
# setup args
args0 = [
"""${ms.MS_TDL} ${ms.CHAN_TDL} ${lsm.LSM_TDL} ms_sel.ms_ifr_subset_str=${ms.IFRS}
ms_sel.input_column=$input_column
ms_sel.output_column=$output_column
stefcal_gain.enabled=1 stefcal_diffgain.enabled=%d %s""" %
((1 if diffgains else 0), STEFCAL_TDLOPTS)
]
if diffgains:
if diffgains is True:
diffgains = "=dE"
args0.append(
"de_subset.subset_enabled=1 de_subset.source_subset=$diffgains")
opts = {
'do_output':
output,
'calibrate_ifrs':
calibrate_ifrs,
'stefcal_gain.mode':
"apply" if apply_only or gain_apply_only else "solve-save",
'stefcal_gain1.mode':
"apply" if apply_only or gain_apply_only else "solve-save",
'stefcal_gain.reset':
int(reset or gain_reset),
'stefcal_diffgain.mode':
"apply" if apply_only or diffgain_apply_only else "solve-save",
'stefcal_diffgain1.mode':
"apply" if apply_only or diffgain_apply_only else "solve-save",
'stefcal_diffgain.reset':
int(reset or diffgain_reset),
'stefcal_diffgain1.reset':
int(reset or diffgain_reset),
'stefcal_ifr_gain_mode':
"apply" if apply_only or ifrgain_apply_only else "solve-save",
'stefcal_ifr_gain_reset':
int(reset or ifrgain_reset),
'stefcal_gain.table':
STEFCAL_GAIN,
'stefcal_gain1.table':
STEFCAL_GAIN1,
'stefcal_diffgain.table':
STEFCAL_DIFFGAIN,
'stefcal_diffgain1.table':
STEFCAL_DIFFGAIN1,
'stefcal_ifr_gain_table':
STEFCAL_IFRGAIN,
'stefcal_visualize':
False
}
# set gain parameters
if gain_smoothing or STEFCAL_GAIN_SMOOTHING or gain_intervals or STEFCAL_GAIN_INTERVALS:
timesmooth, freqsmooth = gain_smoothing or STEFCAL_GAIN_SMOOTHING or (
0, 0)
timeint, freqint = gain_intervals or STEFCAL_GAIN_INTERVALS or (0, 0)
opts['stefcal_gain.timeint'] = 0 if timesmooth else timeint
opts['stefcal_gain.freqint'] = 0 if freqsmooth else freqint
opts['stefcal_gain.timesmooth'] = timesmooth
opts['stefcal_gain.freqsmooth'] = freqsmooth
# set diffgain parameters
if diffgain_smoothing or STEFCAL_DIFFGAIN_SMOOTHING or diffgain_intervals or STEFCAL_DIFFGAIN_INTERVALS:
timesmooth, freqsmooth = diffgain_smoothing or STEFCAL_DIFFGAIN_SMOOTHING or (
0, 0)
timeint, freqint = diffgain_intervals or STEFCAL_DIFFGAIN_INTERVALS or (
0, 0)
opts['stefcal_diffgain.timeint'] = 0 if timesmooth else timeint
opts['stefcal_diffgain.freqint'] = 0 if freqsmooth else freqint
opts['stefcal_diffgain.timesmooth'] = timesmooth
opts['stefcal_diffgain.freqsmooth'] = freqsmooth
# add user-defined args
args0 += list(args)
opts.update(options)
opts.update(kws)
# run the job
mqt.run(STEFCAL_SCRIPT,
STEFCAL_JOBNAME,
section=section,
saveconfig=saveconfig,
args=args0,
options=opts)
# copy gains
try:
if not apply_only:
if os.path.exists(STEFCAL_GAIN) and not gain_apply_only:
std.copy(STEFCAL_GAIN, STEFCAL_GAIN_SAVE)
if gain_plot_prefix:
make_gain_plots(STEFCAL_GAIN_SAVE, prefix=gain_plot_prefix)
if os.path.exists(STEFCAL_GAIN1) and not gain_apply_only:
std.copy(STEFCAL_GAIN1, STEFCAL_GAIN1_SAVE)
if gain_plot_prefix:
make_gain_plots(STEFCAL_GAIN1_SAVE,
prefix=gain_plot_prefix)
if os.path.exists(STEFCAL_DIFFGAIN) and not diffgain_apply_only:
std.copy(STEFCAL_DIFFGAIN, STEFCAL_DIFFGAIN_SAVE)
if diffgain_plot_prefix:
make_diffgain_plots(STEFCAL_DIFFGAIN_SAVE,
prefix=diffgain_plot_prefix)
if os.path.exists(STEFCAL_IFRGAIN) and not ifrgain_apply_only:
std.copy(STEFCAL_IFRGAIN, STEFCAL_IFRGAIN_SAVE)
if ifrgain_plot_prefix:
make_ifrgain_plots(STEFCAL_IFRGAIN_SAVE,
prefix=ifrgain_plot_prefix)
except:
traceback.print_exc()
plotfail("plot routine failed, see exception above")
# post-calibration flagging
if flag_threshold:
if isinstance(flag_threshold, (list, tuple)):
t0, t1 = flag_threshold
else:
t0, t1 = flag_threshold, None
ms.flagms("--above %g" % t0, "-f threshold -c")
if t1:
ms.flagms("--fm-above %g" % t1, "-f fmthreshold -c")
# plot residuals
if plotvis:
try:
info("Plotting visibilities ($plotvis)")
ms.PLOTVIS = plotvis
ms.plotms("-o ${OUTFILE}_${output}${_s<STEP}${_<label}.png")
except:
traceback.print_exc()
plotfail("plot routine failed, see exception above")
# make images
im.make_image(msname,
column=STEFCAL_OUTPUT_COLUMN,
dirty=dirty,
restore=restore,
restore_lsm=restore_lsm)
def azishe(config='$CFG'):
# Get parameters from json file
with open(II(config)) as jsn_std:
jparams = json.load(jsn_std)
# Remove empty strings and convert unicode characters to strings
params = {}
for key, val in jparams.iteritems():
# Make sure all keys are strings
_key = str(key)
# ignore empty strings and comments
if val=="" or _key=="#":
pass
# convert unicode values to strings
elif isinstance(val,unicode):
params[_key] = str(val).lower()
else:
params[_key] = val
get_opts = lambda prefix: filter(lambda a: a[0].startswith(prefix), params.items())
# Retrieve MS and imager options
ms_dict = dict([(key.split('ms_')[-1],val) for (key,val) in get_opts('ms_') ] )
im_dict = dict([(key.split('im_')[-1],val) for (key,val) in get_opts('im_') ] )
# Seperate deconvolution settings
_deconv = {}
for dcv in 'lwimager wsclean moresane casa'.split():
if params[dcv]:
_deconv.update( {dcv:dict([(key.split(dcv+'_')[-1],val) for (key,val) in get_opts(dcv+'_') ] )} )
# Set imaging options
im.IMAGER = imager.IMAGER = params['imager'].lower()
for opt in 'npix weight robust mode stokes'.split():
if opt in im_dict.keys():
setattr(im,opt,im_dict.pop(opt))
im.cellsize = '%farcsec'%(im_dict.pop('cellsize'))
im.stokes = im.stokes.upper()
weight_fov = im_dict.pop('weight_fov')
if weight_fov:
im.weight_fov = '%farcmin'%weight_fov
# Create empty MS
synthesis = ms_dict.pop('synthesis')
scalenoise = 1
if synthesis > 12:
scalenoise = math.sqrt(12.0/synthesis)
msname = II('rodrigues%f.MS'%(time.time()))
obs = params['observatory'].lower()
antennas = _OBS[obs]
freq0 = ms_dict.pop('freq0')*1e6
dfreq = ms_dict.pop('dfreq')*1e3
direction = "J2000,%s,%s"%(ms_dict.pop('ra'),ms_dict.pop('dec'))
ms.create_empty_ms(msname=msname,synthesis=synthesis,freq0=freq0,
dfreq=dfreq,tel=obs,pos='%s/%s'%(OBSDIR,antennas),
direction=direction,**ms_dict)
if exists(msname):
v.MS = msname
else:
abort("Something went wrong while creating the empty MS. Please check logs for details")
makedir(DESTDIR)
ms.plot_uvcov(ms=.1,width=10,height=10,dpi=150,save="$OUTFILE-uvcov.png")
# Set noise for simulation
spwtab = ms.ms(subtable="SPECTRAL_WINDOW")
freq0 = spwtab.getcol("CHAN_FREQ")[ms.SPWID,0]
spwtab.close()
if params['add_noise']:
noise = compute_vis_noise(sefd=get_sefd(freq0)) * scalenoise
else:
noise = 0
# Simulate Visibilities
_katalog = params['katalog_id']
if _katalog:
katalog = '%s/%s'%(KATDIR,_KATALOG[params['katalog_id']])
lsmname = II('${OUTDIR>/}${MS:BASE}.lsm.html')
radius = float(params['radius'])
fluxrange = params['fluxrange'].split('-')
if len(fluxrange)>1:
fluxrange = map(float,fluxrange)
elif len(fluxrange)==1:
fluxrange = [0,float(fluxrange[0])]
select = ''
fits = verify_sky(katalog) == 'FITS'
if radius or fluxrange:
if radius: select += '--select="r<%fdeg" '%radius
if fluxrange:
select += '--select="I<%f" '%fluxrange[1]
select += '--select="I>%f" '%fluxrange[0]
if not fits:
x.sh('tigger-convert $select --recenter=$direction $katalog $lsmname -f')
else:
from pyrap.measures import measures
dm = measures()
direction = dm.direction('J2000',ms_opts[ra],ms_opts[dec])
ra = np.rad2deg(direction['m0']['value'])
dec = np.rad2deg(direction['m1']['value'])
hdu = pyfits.open(temp_file)
hdu[0].hdr['CRVAL1'] = ra
hdu[0].hdr['CRVAL2'] = dec
hdu.writeto(tmp_file,clobber=True)
simsky(lsmname=lsmname,tdlsec='turbo-sim:custom',noise=noise,column='CORRECTED_DATA')
skymodel = params['sky_model']
if skymodel:
simsky(lsmname=skymodel,noise=0 if katalog else noise,addToCol='CORRECTED_DATA')
## Finally Lets image
# make dirty map
im.make_image(psf=True,**im_dict)
# Deconvolve
for deconv in _deconv:
deconv = deconv.lower()
if deconv in STAND_ALONE_DECONV:
im.make_image(algorithm=deconv,restore=_deconv[deconv],restore_lsm=False,**im_dict)
else:
im.IMAGER = deconv
im.make_image(dirty=False,restore=_deconv[deconv],restore_lsm=False,**im_dict)
xo.sh('tar -czvf ${OUTDIR>/}${MS:BASE}.tar.gz $msname')
def stefcal ( msname="$MS",section="$STEFCAL_SECTION",
diffgains=None,
apply_only=False,
reset=False,
gain_apply_only=False,
gain_reset=False,
diffgain_apply_only=False,
diffgain_reset=False,
gain_plot_prefix="$STEFCAL_GAIN_PLOT_PREFIX",
gain1_plot_prefix="$STEFCAL_GAIN1_PLOT_PREFIX",
ifrgain_plot_prefix="$STEFCAL_IFRGAIN_PLOT_PREFIX",
diffgain_plot_prefix="$STEFCAL_DIFFGAIN_PLOT_PREFIX",
ifrgain_apply_only=False,
ifrgain_reset=False,
gain_intervals=None,gain_smoothing=None,
diffgain_intervals=None,diffgain_smoothing=None,
flag_threshold=None,
calibrate_ifrs="$STEFCAL_CALIBRATE_IFRS",
input_column="$STEFCAL_INPUT_COLUMN",
output_column="$STEFCAL_OUTPUT_COLUMN",
output="CORR_RES",
plotvis="${ms.PLOTVIS}",
dirty=True,restore=False,restore_lsm=True,
label=None,
saveconfig="$STEFCAL_SAVE_CONFIG",
plotfail=None,
args=[],options={},
**kws):
"""Generic function to run a stefcal job.
'section' TDL config file section
'label' will be assigned to the global LABEL for purposes of file naming
'apply_only' if true, will only apply saved solutions rather than re-solve
'{gain,diffgain,ifrgain}_apply_only'
if true, will only apply saved gain/diffgain/IFR gain solutions rather than re-solve
'reset' if true, will reset all saved solutuions prior to starting
'{gain,diffgain,ifrgain}_reset'
if true, will reset saved gain/diffgain/IFR gain solutuions prior to starting
'diffgains' set to a source subset string to solve for diffgains. Set to True to use "=dE"
'diffgain_mode' 'solve-save' to solve & save, 'solve-nosave' to not save, 'apply' to apply only
'diffgain_plot' automatically invoke make_diffgain_plots() if True
'flag_threshold' threshold flaging post-solutions. Give one threshold to flag with --above,
or T1,T2 for --above T1 --fm-above T2
'output' output visibilities ('CORR_DATA','CORR_RES', 'RES' are useful)
'plotvis' if not empty, specifies which output visibilities to plot using plot-ms (see plot.ms.py --help)
'dirty','restore'
'restore_lsm' image output visibilities (passed to imager.make_image above as is)
'plotfail' plotting failure reported via warn or abort. Default is warn, set to abort to abort.
'saveconfig' saves the effective TDL config to file[:section]
'args','options' passed to the stefcal job as is (as a list of arguments and kw=value pairs),
can be used to supply extra TDL options
extra keywords: passed to the stefcal job as kw=value, can be used to supply extra TDL options, thus
overriding settings in the TDL config file. Useful arguments of this kind are e.g.:
stefcal_reset_all=True to remove prior gains solutions.
"""
msname,section,lsm,label,plotvis,calibrate_ifrs, \
gain_plot_prefix,gain1_plot_prefix,ifrgain_plot_prefix,diffgain_plot_prefix,saveconfig,plotfail,input_column,output_column = \
interpolate_locals("msname section lsm label plotvis calibrate_ifrs gain_plot_prefix gain1_plot_prefix ifrgain_plot_prefix diffgain_plot_prefix "
"saveconfig plotfail input_column output_column")
plotfail = plotfail or warn
makedir(v.DESTDIR);
# increment step counter and assign global label
if label is not None:
v.LABEL = str(label);
if type(v.STEP) is int and STEFCAL_STEP_INCR:
v.STEP += STEFCAL_STEP_INCR;
# setup stefcal options and run
info("Running stefcal ${step <STEP} ${(<LABEL>)}");
# setup args
args0 = [ """${ms.MS_TDL} ${ms.CHAN_TDL} ${lsm.LSM_TDL} ms_sel.ms_ifr_subset_str=${ms.IFRS}
ms_sel.input_column=$input_column
ms_sel.output_column=$output_column
stefcal_gain.enabled=1 stefcal_diffgain.enabled=%d %s"""%
((1 if diffgains else 0),STEFCAL_TDLOPTS) ];
if diffgains:
if diffgains is True:
diffgains = "=dE";
args0.append("de_subset.subset_enabled=1 de_subset.source_subset=$diffgains");
opts = {
'do_output': output,
'calibrate_ifrs': calibrate_ifrs,
'stefcal_gain.mode': "apply" if apply_only or gain_apply_only else "solve-save",
'stefcal_gain1.mode': "apply" if apply_only or gain_apply_only else "solve-save",
'stefcal_gain.reset': int(reset or gain_reset),
'stefcal_diffgain.mode': "apply" if apply_only or diffgain_apply_only else "solve-save",
'stefcal_diffgain1.mode': "apply" if apply_only or diffgain_apply_only else "solve-save",
'stefcal_diffgain.reset': int(reset or diffgain_reset),
'stefcal_diffgain1.reset': int(reset or diffgain_reset),
'stefcal_ifr_gain_mode': "apply" if apply_only or ifrgain_apply_only else "solve-save",
'stefcal_ifr_gain_reset': int(reset or ifrgain_reset),
'stefcal_gain.table': STEFCAL_GAIN,
'stefcal_gain1.table': STEFCAL_GAIN1,
'stefcal_diffgain.table': STEFCAL_DIFFGAIN,
'stefcal_diffgain1.table': STEFCAL_DIFFGAIN1,
'stefcal_ifr_gain_table': STEFCAL_IFRGAIN,
'stefcal_visualize': False
}
# set gain parameters
if gain_smoothing or STEFCAL_GAIN_SMOOTHING or gain_intervals or STEFCAL_GAIN_INTERVALS:
timesmooth,freqsmooth = gain_smoothing or STEFCAL_GAIN_SMOOTHING or (0,0);
timeint,freqint = gain_intervals or STEFCAL_GAIN_INTERVALS or (0,0);
opts['stefcal_gain.timeint'] = 0 if timesmooth else timeint;
opts['stefcal_gain.freqint'] = 0 if freqsmooth else freqint;
opts['stefcal_gain.timesmooth'] = timesmooth;
opts['stefcal_gain.freqsmooth'] = freqsmooth;
# set diffgain parameters
if diffgain_smoothing or STEFCAL_DIFFGAIN_SMOOTHING or diffgain_intervals or STEFCAL_DIFFGAIN_INTERVALS:
timesmooth,freqsmooth = diffgain_smoothing or STEFCAL_DIFFGAIN_SMOOTHING or (0,0);
timeint,freqint = diffgain_intervals or STEFCAL_DIFFGAIN_INTERVALS or (0,0);
opts['stefcal_diffgain.timeint'] = 0 if timesmooth else timeint;
opts['stefcal_diffgain.freqint'] = 0 if freqsmooth else freqint;
opts['stefcal_diffgain.timesmooth'] = timesmooth;
opts['stefcal_diffgain.freqsmooth'] = freqsmooth;
# add user-defined args
args0 += list(args);
opts.update(options);
opts.update(kws);
# run the job
mqt.run(STEFCAL_SCRIPT,STEFCAL_JOBNAME,section=section,saveconfig=saveconfig,args=args0,options=opts);
# copy gains
try:
if not apply_only:
if os.path.exists(STEFCAL_GAIN) and not gain_apply_only:
std.copy(STEFCAL_GAIN,STEFCAL_GAIN_SAVE);
if gain_plot_prefix:
make_gain_plots(STEFCAL_GAIN_SAVE,prefix=gain_plot_prefix);
if os.path.exists(STEFCAL_GAIN1) and not gain_apply_only:
std.copy(STEFCAL_GAIN1,STEFCAL_GAIN1_SAVE);
if gain_plot_prefix:
make_gain_plots(STEFCAL_GAIN1_SAVE,prefix=gain_plot_prefix);
if os.path.exists(STEFCAL_DIFFGAIN) and not diffgain_apply_only:
std.copy(STEFCAL_DIFFGAIN,STEFCAL_DIFFGAIN_SAVE);
if diffgain_plot_prefix:
make_diffgain_plots(STEFCAL_DIFFGAIN_SAVE,prefix=diffgain_plot_prefix);
if os.path.exists(STEFCAL_IFRGAIN) and not ifrgain_apply_only:
std.copy(STEFCAL_IFRGAIN,STEFCAL_IFRGAIN_SAVE);
if ifrgain_plot_prefix:
make_ifrgain_plots(STEFCAL_IFRGAIN_SAVE,prefix=ifrgain_plot_prefix);
except:
traceback.print_exc();
plotfail("plot routine failed, see exception above");
# post-calibration flagging
if flag_threshold:
if isinstance(flag_threshold,(list,tuple)):
t0,t1 = flag_threshold;
else:
t0,t1 = flag_threshold,None;
ms.flagms("--above %g"%t0,"-f threshold -c");
if t1:
ms.flagms("--fm-above %g"%t1,"-f fmthreshold -c");
# plot residuals
if plotvis:
try:
info("Plotting visibilities ($plotvis)");
ms.PLOTVIS = plotvis;
ms.plotms("-o ${OUTFILE}_${output}${_s<STEP}${_<label}.png");
except:
traceback.print_exc();
plotfail("plot routine failed, see exception above");
# make images
im.make_image(msname,column=STEFCAL_OUTPUT_COLUMN,dirty=dirty,restore=restore,restore_lsm=restore_lsm);