def selfcal(msfile,
params,
mode='p',
in_gaintable=[],
out_gaintable='sc_p.test.gcal',
solint='8min'):
refant = params['flagging']['refant']
#print(gaintables, 'before selfcal')
if mode == 'p':
minsnr = 3.0
solnorm = False
elif mode == 'ap':
minsnr = 3.0
solnorm = True
cts.gaincal(msfile,
caltable=out_gaintable,
append=False,
field='0',
spw='0',
uvrange='',
solint=solint,
refant=refant,
minsnr=minsnr,
gaintype='G',
solnorm=solnorm,
calmode=mode,
solmode='R',
gaintable=in_gaintable,
interp=['nearestobsid, nearestobsid'],
parang=False)
return None
def kc_cal(msin, model_cl):
"""Get gain and delay calibration solutions
:param msin: Visibility dataset in measurement set format path
:type msin: str
:param model_cl: Point source model path
:type model_cl: str
:return: Calibration solutions tables
:rtype: list
"""
# Fill the model column
ft(msin, complist=model_cl, usescratch=True)
kc = calname(msin, 'K') # Delays
gc = calname(msin, 'G') # Gains
gaincal(vis=msin,
caltable=kc,
gaintype='K',
solint='inf',
refant='11',
minsnr=1)
# Ensure reference antenna exists and isn't faulty
gaincal(vis=msin,
caltable=gc,
gaintype='G',
solint='inf',
refant='11',
minsnr=1,
calmode='ap',
gaintable=kc)
applycal(msin, gaintable=[kc, gc])
return [kc, gc]
def apcal(msfile, params, field=None):
refant = params['flagging']['refant']
gaintables = params['calibration']['gaintables']
apgaintable = params['calibration']['apgaintable']
uvran = params['general']['uvran']
spw = params['general']['spw']
print(gaintables, 'before ampcal')
cts.gaincal(vis=msfile,
caltable=apgaintable,
spw=spw,
append=True,
field=field,
uvrange=uvran,
solint='60s',
refant=refant,
minsnr=3.0,
gaintype='G',
calmode='ap',
solmode='R',
gaintable=gaintables,
interp=['linear, linear', 'linear, linear'],
parang=True)
return print('Amplitude and phase calibration done for ', field)
def bpasscal(msfile, params, field=None):
refant = params['flagging']['refant']
bpassgaintable = params['calibration']['bpassgaintable']
bpasstable = params['calibration']['bpasstable']
gaintables = params['calibration']['gaintables']
uvran = params['general']['uvran']
ch0_spw = params['calibration']['ch0_spw']
# Initial gain calibration
cts.gaincal(msfile,
caltable=bpassgaintable,
field=field,
uvrange=uvran,
spw='0',
solint='int',
solmode='R',
refant=refant,
minsnr=2.0,
gaintype='G',
calmode='ap',
gaintable=gaintables,
interp=[
'linear, linear',
],
append=True,
parang=False)
gaintables.append(bpassgaintable)
print(gaintables, 'before bpass')
# bpass calibration
cts.bandpass(msfile,
caltable=bpasstable,
spw='0',
field=field,
uvrange=uvran,
solint='inf',
refant=refant,
solnorm=True,
minsnr=2.0,
append=True,
fillgaps=8,
parang=False,
gaintable=gaintables,
interp=['linear,linear', 'linear,linear'])
#cts.bandpass(msfile, caltable=bpasstable, spw ='0', field=field, solint='inf', refant=refant, solnorm = True, minsnr=2.0, fillgaps=8, bandtype='BPOLY', degamp=5, degphase=7, parang = True, gaintable=gaintables, interp=['nearest,nearestflag','nearest,nearestflag'])
# Removing and also deleting the on-the fly gaincal table used for bandpass
subprocess.run('rm -r {}'.format(bpassgaintable), shell=True, check=True)
gaintables.remove(bpassgaintable)
return print('Bandpass calibration done')
def delaycal(msfile, params, field=None):
refant = params['flagging']['refant']
caltable = params['calibration']['delaytable']
gaintables = params['calibration']['gaintables']
uvran = params['general']['uvran']
cts.gaincal(msfile,
field=field,
solint='60s',
spw='0',
uvrange=uvran,
refant=refant,
caltable=caltable,
gaintype='K',
solnorm=True,
gaintable=gaintables,
parang=True)
return print('Delay calibration done')
def self_calibration_from_final_pcal_model(project: project.Project,
src_model: Path):
raise NotImplementedError
# Assumes the model is a CASA image
for a_cal in project.phase_calibrators:
calsc = project.caldir / f"{project.project_name.lower()}.{a_cal}.sc_final"
# TODO: check if ft is in casatools or casatasks
casatools.ft(vis=str(project.msfile),
field=a_cal,
model=str(src_model),
usescratch=True)
casatasks.gaincal(vis=str(project.msfile),
caltable=str(calsc),
field=a_cal,
solint='3min',
refant=','.join(project.refants),
gaintype='G',
calmode='ap',
interp=['nearest'],
parang=True)
casatasks.applycal()
def run_all_uvstats(myvis,
out_path,
uv_threshold=3,
uv_nsigma=3,
try_phase_selfcal=True,
cleanup_calsplit=True,
cleanup_phaseselfcal=True,
remake_split=True):
if not os.path.isdir(out_path):
os.mkdir(out_path)
from casatasks import split, gaincal, applycal
from casaplotms import plotms
# from taskinit import msmdtool, casalog
# from taskinit import msmdtool, casalog
from casatools import ms
myms = ms()
# msmd = msmdtool()
# get metadata
myms.open(myvis)
mymsmd = myms.metadata()
# mymsmd.open(myvis)
cal_fields = np.unique(mymsmd.fieldsforintent('CALIBRATE*'))
field_names = mymsmd.namesforfields(cal_fields)
myms.close()
# split calibrator visibilities
field_str = ','.join([str(f) for f in cal_fields])
output_cal_ms = out_path + '/cal_fields.ms'
if os.path.exists(output_cal_ms) and remake_split:
os.system('rm -r {0}'.format(output_cal_ms))
if not os.path.exists(output_cal_ms):
split(vis=myvis,
field=field_str,
keepflags=True,
timebin='0s',
outputvis=output_cal_ms)
# There are the flux cals which have built in models in CASA.
skip_fields = ['3C286', '3C48', '3C147', '3C138']
gaincal_tables = []
for field_name in field_names:
if np.any([field_name in skip1 for skip1 in skip_fields]):
continue
plotms_outfile = out_path + '/plotms_amp_uvwave_field_{0}.txt'.format(
field_name)
casalog.post(
message='Exporting from plotms: {0}'.format(plotms_outfile),
origin='run_all_uvstats')
if not os.path.exists(plotms_outfile):
plotms(vis=out_path + '/cal_fields.ms',
field=field_name,
xaxis='UVwave',
yaxis='Amp',
ydatacolumn='data',
averagedata=True,
scalar=False,
avgchannel='4096',
avgtime='1000',
avgscan=False,
correlation='RR,LL',
plotfile=plotms_outfile,
showgui=False,
overwrite=True)
else:
casalog.post(message='File {0} already exists. Skipping'.format(
plotms_outfile),
origin='run_all_uvstats')
infile = out_path + '/plotms_amp_uvwave_field_{0}.txt'.format(
field_name)
casalog.post(message='Analyzing UV stats for {0}'.format(infile),
origin='run_all_uvstats')
# Read in txt from plotms
dat, median_flux = get_uvdata(infile)
# n_scans = len(np.unique(dat['scan']))
binned_dat = bin_uvdata(dat)
binned_dat_perscan = bin_uvdata_perscan(dat)
if binned_dat.shape[1] == 0:
continue
if binned_dat_perscan.shape[1] == 0:
continue
plot_uvdata_perscan(binned_dat_perscan,
binned_dat,
infile,
bin_type='combined')
# try phase-only selfcal
if try_phase_selfcal:
gaincal_table = out_path + '/cal_field_{0}.g'.format(field_name)
try:
gaincal(vis=output_cal_ms,
caltable=gaincal_table,
field=field_name,
solint='int',
refant='',
calmode='p')
applycal(vis=output_cal_ms,
gaintable=gaincal_table,
field=field_name,
calwt=False)
gaincal_tables.append(gaincal_table)
plotms_outfile = out_path + '/plotms_amp_uvwave_cal_field_{0}.txt'.format(
field_name)
casalog.post(message='Exporting from plotms: {0}'.format(
plotms_outfile),
origin='run_all_uvstats')
if not os.path.exists(plotms_outfile):
plotms(vis=output_cal_ms,
field=field_name,
xaxis='UVwave',
yaxis='Amp',
ydatacolumn='corrected',
averagedata=True,
scalar=False,
avgchannel='4096',
avgtime='1000',
avgscan=False,
correlation='RR,LL',
plotfile=plotms_outfile,
showgui=False,
overwrite=True)
else:
casalog.post(
message='File {0} already exists. Skipping'.format(
plotms_outfile),
origin='run_all_uvstats')
infile = out_path + '/plotms_amp_uvwave_cal_field_{0}.txt'.format(
field_name)
casalog.post(
message='Analyzing UV stats for {0}'.format(infile),
origin='run_all_uvstats')
dat, median_flux = get_uvdata(infile)
# n_scans = len(np.unique(dat['scan']))
binned_dat = bin_uvdata(dat)
binned_dat_perscan = bin_uvdata_perscan(dat)
if binned_dat.shape[1] == 0:
continue
if binned_dat_perscan.shape[1] == 0:
continue
plot_uvdata_perscan(binned_dat_perscan,
binned_dat,
infile,
bin_type='combined')
except:
casalog.post(
message='Problem calibrating field {0}'.format(field_name),
origin='run_all_uvstats')
# Delete calibrator split
if cleanup_calsplit:
os.system("rm -r {}".format(output_cal_ms))
os.system("rm -r {}.flagversions".format(output_cal_ms))
# Delete gaincal tables
if cleanup_phaseselfcal:
for gaincal_table in gaincal_tables:
os.system("rm -r {}".format(gaincal_table))
def slefcal_ms(src_dir, imagems, imagename, ATCA_band, n_spw):
print(
"+ + + + + + + + + + + + + + + + +\n+ Self Cal Round 1 +\n+ + + + + + + + + + + + + + + + +"
)
mode = "mfs"
nterms = 2
niter = 3000
antenna = "0~6,0~6"
n_spw = n_spw
if ATCA_band == "L":
imsize = 2250
solint = "60s"
minsnr = 3.0
cell = "1arcsec"
minblperant = 3
if ATCA_band == "C":
imsize = 2250
solint = "60s"
minsnr = 3.0
minblperant = 3
cell = "0.5arcsec"
if ATCA_band == "X":
imsize = 1152
solint = "60s"
minsnr = 3.0
minblperant = 3
cell = "0.2arcsec"
uvrange = ""
stokes = "I"
weighting = "briggs"
robust = 0.5
interactive = False
gain = 0.01
threshold = "5e-4Jy"
# if os.path.exists(f"{src_dir}/cal_tables/pcal1_{imagename}"):
rmtables(f"{src_dir}/cal_tables/pcal1_{imagename}")
gaincal(
vis=imagems,
caltable=f"{src_dir}/cal_tables/pcal1_{imagename}",
combine="scan,spw",
spwmap=[0] * n_spw,
gaintype="G",
calmode="p",
solint=solint,
minsnr=minsnr,
minblperant=minblperant,
)
applycal(
vis=imagems,
gaintable=f"{src_dir}/cal_tables/pcal1_{imagename}",
spwmap=[0] * n_spw,
parang=True,
applymode="calonly",
flagbackup=False,
)
flagmanager(vis=imagems, mode="save", versionname="post self1")
for i in range(0, n_spw):
spw = str(i)
# if os.path.exists(f"{src_dir}/casa_files/{imagename}_{spw}_self1*"):
os.system(
f"rm -r {src_dir}/casa_files/{imagename}_{spw}_self1*"
)
print("Cleaning on band: " + str(spw))
tclean(
vis=imagems,
imagename=f"{src_dir}/casa_files/{imagename}_{spw}_self1",
selectdata=True,
mask=f"{src_dir}/casa_files/{imagename}_mfs.mask",
spw=spw,
startmodel=f"{src_dir}/casa_files/{imagename}_{spw}_preself.model",
gain=gain,
specmode=mode,
nterms=nterms,
niter=niter,
threshold=threshold,
imsize=imsize,
cell=cell,
stokes=stokes,
weighting=weighting,
robust=robust,
antenna=antenna,
interactive=interactive,
savemodel="modelcolumn",
pbcor=False,
uvrange=uvrange,
)
tclean(
vis=imagems,
imagename=f"{src_dir}/casa_files/{imagename}_{spw}_self1",
selectdata=True,
mask="",
spw=spw,
gain=gain,
specmode=mode,
nterms=nterms,
niter=0,
threshold=threshold,
imsize=imsize,
cell=cell,
stokes=stokes,
weighting=weighting,
robust=robust,
antenna=antenna,
interactive=interactive,
pbcor=False,
savemodel="modelcolumn",
calcres=False,
calcpsf=False,
uvrange=uvrange,
)
threshold = "5e-5Jy"
# if os.path.exists(f"{src_dir}/cal_tables/pcal2_{imagename}"):
rmtables(f"{src_dir}/cal_tables/pcal2_{imagename}")
print(
"+ + + + + + + + + + + + + + + + +\n+ Self Cal Round 2 +\n+ + + + + + + + + + + + + + + + +"
)
gaincal(
vis=imagems,
caltable=f"{src_dir}/cal_tables/pcal2_{imagename}",
gaintable=f"{src_dir}/cal_tables/pcal1_{imagename}",
combine="scan,spw",
spwmap=[0] * n_spw,
gaintype="G",
calmode="p",
solint=solint,
minsnr=minsnr,
minblperant=minblperant,
)
applycal(
vis=imagems,
gaintable=[
f"{src_dir}/cal_tables/pcal1_{imagename}",
f"{src_dir}/cal_tables/pcal2_{imagename}",
],
spwmap=[[0] * n_spw, [0] * n_spw],
parang=True,
applymode="calonly",
flagbackup=False,
)
flagmanager(vis=imagems, mode="save", versionname="post self2")
for i in range(0, n_spw):
spw = str(i)
# if os.path.exists(f"{src_dir}/casa_files/{imagename}_{spw}_self2*"):
os.system(f"rm -r {src_dir}/casa_files/{imagename}_{spw}_self2*")
print("Cleaning on band: " + str(spw))
tclean(
vis=imagems,
imagename=f"{src_dir}/casa_files/{imagename}_{spw}_self2",
selectdata=True,
mask=f"{src_dir}/casa_files/{imagename}_mfs.mask",
spw=spw,
startmodel=f"{src_dir}/casa_files/{imagename}_{spw}_self1.model",
gain=gain,
specmode=mode,
nterms=nterms,
niter=niter,
threshold=threshold,
imsize=imsize,
cell=cell,
stokes=stokes,
weighting=weighting,
robust=robust,
antenna=antenna,
interactive=interactive,
savemodel="modelcolumn",
pbcor=False,
uvrange=uvrange,
)
tclean(
vis=imagems,
imagename=f"{src_dir}/casa_files/{imagename}_{spw}_self2",
selectdata=True,
mask="",
spw=spw,
gain=gain,
specmode=mode,
nterms=nterms,
niter=0,
threshold=threshold,
imsize=imsize,
cell=cell,
stokes=stokes,
weighting=weighting,
robust=robust,
antenna=antenna,
interactive=interactive,
pbcor=False,
savemodel="modelcolumn",
calcres=False,
calcpsf=False,
uvrange=uvrange,
)
threshold = "5e-6Jy"
print(
"+ + + + + + + + + + + + + + + + +\n+ Self Cal Round 3 +\n+ + + + + + + + + + + + + + + + +"
)
rmtables(f"{src_dir}/cal_tables/pcal3_{imagename}")
gaincal(
vis=imagems,
caltable=f"{src_dir}/cal_tables/pcal3_{imagename}",
gaintable=[
f"{src_dir}/cal_tables/pcal1_{imagename}",
f"{src_dir}/cal_tables/pcal2_{imagename}",
],
combine="scan,spw",
spwmap=[[0] * n_spw, [0] * n_spw],
gaintype="G",
calmode="p",
solint=solint,
minsnr=minsnr,
minblperant=minblperant,
)
applycal(
vis=imagems,
gaintable=[
f"{src_dir}/cal_tables/pcal1_{imagename}",
f"{src_dir}/cal_tables/pcal2_{imagename}",
f"{src_dir}/cal_tables/pcal3_{imagename}",
],
spwmap=[[0] * n_spw, [0] * n_spw, [0] * n_spw],
parang=True,
applymode="calonly",
flagbackup=False,
)
flagmanager(vis=imagems, mode="save", versionname="post self3")
for i in range(0, n_spw):
spw = str(i)
# if os.path.exists(f"{src_dir}/casa_files/{imagename}_{spw}_self3*"):
os.system(f"rm -r {src_dir}/casa_files/{imagename}_{spw}_self3*")
print("Cleaning on band: " + str(spw))
# imagename = f"{src_dir}/casa_files/{imagename}_{spw}"
tclean(
vis=imagems,
imagename=f"{src_dir}/casa_files/{imagename}_{spw}_self3",
selectdata=True,
mask=f"{src_dir}/casa_files/{imagename}_mfs.mask",
startmodel=f"{src_dir}/casa_files/{imagename}_{spw}_self2.model",
spw=spw,
gain=gain,
specmode=mode,
nterms=nterms,
niter=niter,
threshold=threshold,
imsize=imsize,
cell=cell,
stokes=stokes,
weighting=weighting,
robust=robust,
antenna=antenna,
interactive=interactive,
savemodel="modelcolumn",
pbcor=False,
uvrange=uvrange,
)
tclean(
vis=imagems,
imagename=f"{src_dir}/casa_files/{imagename}_{spw}_self3",
selectdata=True,
mask="",
spw=spw,
gain=gain,
specmode=mode,
nterms=nterms,
niter=0,
threshold=threshold,
imsize=imsize,
cell=cell,
stokes=stokes,
weighting=weighting,
robust=robust,
antenna=antenna,
interactive=interactive,
pbcor=False,
savemodel="modelcolumn",
calcres=False,
calcpsf=False,
uvrange=uvrange,
)
return
def calibrate_ms(src_dir, msname, ATCA_band, ref, pri, sec, tar):
os.system(f"rm -r {src_dir}/cal_tables/cal_{pri}_{ATCA_band}.F0")
setjy(
vis=msname,
field=pri,
scalebychan=True,
standard="Perley-Butler 2010",
usescratch=True,
)
print(f"Performing gain calibration on {pri}")
gaincal(
vis=msname,
caltable=f"{src_dir}/cal_tables/cal_{pri}_{ATCA_band}.G0",
field=pri,
refant=ref,
gaintype="G",
calmode="p",
parang=True,
# minblperant=3,
solint="60s",
)
print(f"Performing bandpass calibration on {pri}")
bandpass(
vis=msname,
caltable=f"{src_dir}/cal_tables/cal_{pri}_{ATCA_band}.B0",
field=pri,
refant=ref,
solnorm=True,
solint="120s",
bandtype="B",
gaintable=[f"{src_dir}/cal_tables/cal_{pri}_{ATCA_band}.G0"],
parang=True,
)
print(f"Determining gains on {sec}")
gaincal(
vis=msname,
caltable=f"{src_dir}/cal_tables/cal_{pri}_{ATCA_band}.G1",
field=pri + "," + sec,
refant=ref,
gaintype="G",
calmode="ap",
parang=True,
solint="120s",
gaintable=[f"{src_dir}/cal_tables/cal_{pri}_{ATCA_band}.B0"],
)
bandpass(
vis=msname,
caltable=f"{src_dir}/cal_tables/cal_{pri}_{ATCA_band}.B1",
field=pri,
refant=ref,
solnorm=True,
solint="120s",
bandtype="B",
gaintable=[f"{src_dir}/cal_tables/cal_{pri}_{ATCA_band}.G1"],
parang=True,
)
print(f"Deriving gain calibration using {pri}")
gaincal(
vis=msname,
caltable=f"{src_dir}/cal_tables/cal_{pri}_{ATCA_band}.G2",
field=pri,
refant=ref,
gaintype="G",
calmode="ap",
parang=True,
solint="60s",
# minblperant=3,
gaintable=[f"{src_dir}/cal_tables/cal_{pri}_{ATCA_band}.B1"],
)
print(f"Deriving gain calibration using {sec}")
gaincal(
vis=msname,
caltable=f"{src_dir}/cal_tables/cal_{pri}_{ATCA_band}.G2",
field=sec,
refant=ref,
gaintype="G",
calmode="ap",
parang=True,
solint="60s",
# minblperant=3,
gaintable=[f"{src_dir}/cal_tables/cal_{pri}_{ATCA_band}.B1"],
append=True,
)
print(
"Correcting the flux scale using comparison between the primary and secondary calibrator."
)
fluxscale(
vis=msname,
caltable=f"{src_dir}/cal_tables/cal_{pri}_{ATCA_band}.G2",
fluxtable=f"{src_dir}/cal_tables/cal_{pri}_{ATCA_band}.F0",
reference=pri,
)
flagmanager(vis=msname, mode="save", versionname="before_applycal")
return
def bpasscal2(msfile, params, field=None):
refant = params['flagging']['refant']
bpassgaintable = params['calibration']['bpassgaintable']
bpasstable = params['calibration']['bpasstable']
gaintables = params['calibration']['gaintables']
uvran = params['general']['uvran']
ch0_spw = params['calibration']['ch0_spw']
ini_fgloops = params['calibration']['ini_nloops']
fluxcal = params['general']['fluxcal']
# Initial gain calibration
# for i in range(ini_fgloops):
print('Running the initial gaincal pre-bandpass ...')
cts.gaincal(msfile,
caltable=bpassgaintable,
field=field,
uvrange=uvran,
spw=ch0_spw,
solint='120s',
solmode='R',
refant=refant,
minsnr=2.0,
gaintype='G',
calmode='p',
gaintable=gaintables,
interp=[
'linear, linear',
],
append=True,
parang=False)
'''
for j in fluxcal:
#print('Applying the calibration table to ', i)
cts.applycal(msfile, field=j, spw=ch0_spw, gaintable=bpassgaintable, gainfield=[
'', '', j], uvrange=uvran, interp=['', '', 'linear'], calwt=[False], parang=False)
#cts.flagdata(msfile, mode='tfcrop', datacolumn='corrected', field=j, ntime='scan', spw=ch0_spw,
# timecutoff=6, freqcutoff=6, timefit='poly',freqfit='line',flagdimension='time',
# extendflags=False, timedevscale=5.0,freqdevscale=5.0, extendpols=False,growaround=False,
# action='apply', flagbackup=True,overwrite=True, writeflags=True)
#cts.flagdata(msfile, params, field=j, tcut=6, fcut=6, instance='postcal')
#flg.extend(msfile, params, field=j, grow=80, instance='postcal')
cts.clearcal(msfile) # Removes the corrected datacolumn
if i == (ini_fgloops - 1):
print('Single channel flagcal loops ended.')
break
else:
subprocess.run('rm -r {}'.format(bpassgaintable), shell=True, check=True)
'''
gaintables.append(bpassgaintable)
print(gaintables, 'before bpass')
# bpass calibration
cts.bandpass(msfile,
caltable=bpasstable,
spw='0',
field=field,
uvrange=uvran,
solint='inf',
refant=refant,
solnorm=True,
minsnr=2.0,
append=True,
fillgaps=8,
parang=False,
gaintable=gaintables,
interp=['linear,linear', 'linear,linear'])
#cts.bandpass(msfile, caltable=bpasstable, spw ='0', field=field, solint='inf', refant=refant, solnorm = True, minsnr=2.0, fillgaps=8, bandtype='BPOLY', degamp=5, degphase=7, parang = True, gaintable=gaintables, interp=['nearest,nearestflag','nearest,nearestflag'])
# Removing and also deleting the on-the fly gaincal table used for bandpass
subprocess.run('rm -r {}'.format(bpassgaintable), shell=True, check=True)
gaintables.remove(bpassgaintable)
return print('Bandpass calibration done')