def bandpass_cal(msin):
"""Bandpass calbration
:param msin: Visibility dataset in measurement set format path
:type msin: str
:return: Calibration solutions table
:type: str
"""
bc = calname(msin, 'B')
bandpass(vis=msin, minsnr=1, solnorm=False, bandtype='B', caltable=bc)
applycal(msin, gaintable=[bc])
return bc
def bpasscal(msfile, params, field=None):
refant = params['flagging']['refant']
bpassgaintable = params['calibration']['bpassgaintable']
bpasstable = params['calibration']['bpasstable']
gaintables = params['calibration']['gaintables']
uvran = params['general']['uvran']
ch0_spw = params['calibration']['ch0_spw']
# Initial gain calibration
cts.gaincal(msfile,
caltable=bpassgaintable,
field=field,
uvrange=uvran,
spw='0',
solint='int',
solmode='R',
refant=refant,
minsnr=2.0,
gaintype='G',
calmode='ap',
gaintable=gaintables,
interp=[
'linear, linear',
],
append=True,
parang=False)
gaintables.append(bpassgaintable)
print(gaintables, 'before bpass')
# bpass calibration
cts.bandpass(msfile,
caltable=bpasstable,
spw='0',
field=field,
uvrange=uvran,
solint='inf',
refant=refant,
solnorm=True,
minsnr=2.0,
append=True,
fillgaps=8,
parang=False,
gaintable=gaintables,
interp=['linear,linear', 'linear,linear'])
#cts.bandpass(msfile, caltable=bpasstable, spw ='0', field=field, solint='inf', refant=refant, solnorm = True, minsnr=2.0, fillgaps=8, bandtype='BPOLY', degamp=5, degphase=7, parang = True, gaintable=gaintables, interp=['nearest,nearestflag','nearest,nearestflag'])
# Removing and also deleting the on-the fly gaincal table used for bandpass
subprocess.run('rm -r {}'.format(bpassgaintable), shell=True, check=True)
gaintables.remove(bpassgaintable)
return print('Bandpass calibration done')
def bandpass_with_gap_interpolation_deprecated(myvis,
context,
refantignore="",
search_string="test",
task_string="hifv_testBPdcals"):
'''
Re-do the bandpass accounting for flagged gaps.
Looks for and uses the standard pipeline naming from
the VLA pipeline.
Also see I-Da Chiang's implementation:
https://github.com/z0mgs/EveryTHINGS/blob/master/everythings_bandpass.py
DEPRECATED as of 22/07/20. See new gap interpolation method below.
The `bandpass` task only uses nearest neighbour interpolation across the gap. We want to use
more channels on each side for a better interpolation across the gap.
'''
import pipeline.hif.heuristics.findrefant as findrefant
# Look for BP table
bpname = glob("{0}.{1}.s*_4.{2}BPcal.tbl".format(myvis, task_string,
search_string))
# test and final cal steps will have 1 match:
if len(bpname) == 1:
bpname = bpname[0]
elif len(bpname) == 2:
# One table for each semifinal cal call.
# Grab the second call table.
bpname = sorted(
bpname,
key=lambda x: int(x.split("_4.BPcal.tbl")[0].split(".s")[-1]))[-1]
elif len(bpname) == 0:
raise ValueError("No matches to the BP table found.")
else:
raise ValueError("Found too many matches to the BP table."
" Unclear which should be used: {0}".format(bpname))
# Remove already-made version
# rmtables(bpname)
# Or copy to another name to check against
os.system("mv {0} {0}.orig".format(bpname))
# Get the scan/field selections
scanheur = context.evla['msinfo'][context.evla['msinfo'].keys()[0]]
# Grab list of preferred refants
refantfield = scanheur.calibrator_field_select_string
refantobj = findrefant.RefAntHeuristics(vis=myvis,
field=refantfield,
geometry=True,
flagging=True,
intent='',
spw='',
refantignore=refantignore)
RefAntOutput = refantobj.calculate()
refAnt = ','.join(RefAntOutput)
# Lastly get list of other cal tables to use in the solution
gc_tbl = glob("{}.hifv_priorcals.s*_2.gc.tbl".format(myvis))
assert len(gc_tbl) == 1
opac_tbl = glob("{}.hifv_priorcals.s*_3.opac.tbl".format(myvis))
assert len(opac_tbl) == 1
rq_tbl = glob("{}.hifv_priorcals.s*_4.rq.tbl".format(myvis))
assert len(rq_tbl) == 1
# Check ant correction
ant_tbl = glob("{}.hifv_priorcals.s*_6.ants.tbl".format(myvis))
priorcals = [gc_tbl[0], opac_tbl[0], rq_tbl[0]]
if len(ant_tbl) == 1:
priorcals.extend(ant_tbl)
del_tbl = glob("{0}.{1}.s*_2.{2}delay.tbl".format(myvis, task_string,
search_string))
# test and final cal steps will have 1 match:
if len(del_tbl) == 1:
del_tbl = del_tbl[0]
elif len(del_tbl) == 2:
# One table for each semifinal cal call.
# Grab the second call table.
del_tbl = sorted(
del_tbl,
key=lambda x: int(x.split("_2.delay.tbl")[0].split(".s")[-1]))[-1]
elif len(del_tbl) == 0:
raise ValueError("No matches to the BP table found.")
else:
raise ValueError("Found too many matches to the delay table."
" Unclear which should be used: {0}".format(del_tbl))
# Slight difference in BP initial gain table names
if task_string == "hifv_testBPdcals":
tabname_string = "BPdinitialgain"
else:
tabname_string = "BPinitialgain"
BPinit_tbl = glob("{0}.{1}.s*_3.{2}{3}.tbl".format(myvis, task_string,
search_string,
tabname_string))
# test and final cal steps will have 1 match:
if len(BPinit_tbl) == 1:
BPinit_tbl = BPinit_tbl[0]
elif len(BPinit_tbl) == 2:
# One table for each semifinal cal call.
# Grab the second call table.
BPinit_tbl = sorted(BPinit_tbl,
key=lambda x: int(
x.split("_3.{}.tbl".format(tabname_string))[0].
split(".s")[-1]))[-1]
elif len(BPinit_tbl) == 0:
raise ValueError("No matches to the BP table found.")
else:
raise ValueError(
"Found too many matches to the BP initgain table."
" Unclear which should be used: {0}".format(BPinit_tbl))
gaintables = copy(priorcals)
gaintables.extend([del_tbl, BPinit_tbl])
bandpass(vis=myvis,
caltable=bpname,
field=scanheur.bandpass_field_select_string,
selectdata=True,
scan=scanheur.bandpass_scan_select_string,
solint='inf',
combine='scan',
refant=refAnt,
minblperant=4,
minsnr=5.0,
solnorm=False,
bandtype='B',
smodel=[],
append=False,
fillgaps=400,
docallib=False,
gaintable=gaintables,
gainfield=[''],
interp=[''],
spwmap=[],
parang=True)
def calibrate_ms(src_dir, msname, ATCA_band, ref, pri, sec, tar):
os.system(f"rm -r {src_dir}/cal_tables/cal_{pri}_{ATCA_band}.F0")
setjy(
vis=msname,
field=pri,
scalebychan=True,
standard="Perley-Butler 2010",
usescratch=True,
)
print(f"Performing gain calibration on {pri}")
gaincal(
vis=msname,
caltable=f"{src_dir}/cal_tables/cal_{pri}_{ATCA_band}.G0",
field=pri,
refant=ref,
gaintype="G",
calmode="p",
parang=True,
# minblperant=3,
solint="60s",
)
print(f"Performing bandpass calibration on {pri}")
bandpass(
vis=msname,
caltable=f"{src_dir}/cal_tables/cal_{pri}_{ATCA_band}.B0",
field=pri,
refant=ref,
solnorm=True,
solint="120s",
bandtype="B",
gaintable=[f"{src_dir}/cal_tables/cal_{pri}_{ATCA_band}.G0"],
parang=True,
)
print(f"Determining gains on {sec}")
gaincal(
vis=msname,
caltable=f"{src_dir}/cal_tables/cal_{pri}_{ATCA_band}.G1",
field=pri + "," + sec,
refant=ref,
gaintype="G",
calmode="ap",
parang=True,
solint="120s",
gaintable=[f"{src_dir}/cal_tables/cal_{pri}_{ATCA_band}.B0"],
)
bandpass(
vis=msname,
caltable=f"{src_dir}/cal_tables/cal_{pri}_{ATCA_band}.B1",
field=pri,
refant=ref,
solnorm=True,
solint="120s",
bandtype="B",
gaintable=[f"{src_dir}/cal_tables/cal_{pri}_{ATCA_band}.G1"],
parang=True,
)
print(f"Deriving gain calibration using {pri}")
gaincal(
vis=msname,
caltable=f"{src_dir}/cal_tables/cal_{pri}_{ATCA_band}.G2",
field=pri,
refant=ref,
gaintype="G",
calmode="ap",
parang=True,
solint="60s",
# minblperant=3,
gaintable=[f"{src_dir}/cal_tables/cal_{pri}_{ATCA_band}.B1"],
)
print(f"Deriving gain calibration using {sec}")
gaincal(
vis=msname,
caltable=f"{src_dir}/cal_tables/cal_{pri}_{ATCA_band}.G2",
field=sec,
refant=ref,
gaintype="G",
calmode="ap",
parang=True,
solint="60s",
# minblperant=3,
gaintable=[f"{src_dir}/cal_tables/cal_{pri}_{ATCA_band}.B1"],
append=True,
)
print(
"Correcting the flux scale using comparison between the primary and secondary calibrator."
)
fluxscale(
vis=msname,
caltable=f"{src_dir}/cal_tables/cal_{pri}_{ATCA_band}.G2",
fluxtable=f"{src_dir}/cal_tables/cal_{pri}_{ATCA_band}.F0",
reference=pri,
)
flagmanager(vis=msname, mode="save", versionname="before_applycal")
return
def main_calibration(project: project.Project):
"""Runs the main calibration of the data:
- instrumental delay corrections: in the specified time range.
- global fringe fit: on all calibrators (and target if no phase-referencing is used).
- bandpass calibration: using the bandpass calibrators.
"""
cals = {
'sbd': (project.caldir / f"{project.project_name.lower()}.sbd"),
'mbd': (project.caldir / f"{project.project_name.lower()}.mbd"),
'bp': (project.caldir / f"{project.project_name.lower()}.bp")
}
do_all_exists = []
for a_cal in cals:
do_all_exists.append(cals[a_cal].exists())
if not all(do_all_exists):
casatasks.fringefit(vis=str(project.msfile),
caltable=str(cals['sbd']),
timerange=project.instr_delay_timerange,
solint='inf',
zerorates=True,
refant=','.join(project.refants),
minsnr=50,
docallib=True,
callib=str(project.caldir / "cal_library.txt"),
parang=True)
write_callib(filename=str(project.caldir / "cal_library.txt"),
entry=f"caltable='{cals['sbd']}' tinterp='nearest'")
casatasks.fringefit(vis=str(project.msfile),
caltable=str(cals['mbd']),
field=','.join(project.calibrators),
solint='inf',
zerorates=False,
refant=','.join(project.refants),
combine='spw',
minsnr=5,
docallib=True,
callib=str(project.caldir / "cal_library.txt"),
parang=True)
spw_with_solutions = get_spw_global_fringe(caltable=str(cals['mbd']))
write_callib(filename=str(project.caldir/"cal_library.txt"),
entry=f"caltable='{cals['mbd']}' tinterp='linear' " \
f"spwmap={project.nsubbands}*[{spw_with_solutions}]")
casatasks.bandpass(vis=str(project.msfile),
field=','.join(project.bandpass_calibrators),
combine='scan',
caltable=str(cals['bp']),
docallib=True,
callib=str(project.caldir / "cal_library.txt"),
solnorm=True,
solint='inf',
refant=','.join(project.refants),
bandtype='B',
parang=True,
fillgaps=16)
write_callib(
filename=str(project.caldir / "cal_library.txt"),
entry=f"caltable='{cals['bp']}' tinterp='linear' finterp='linear'")
else:
print(
f"The tables related to the main calibration already exists. They will not be generated again."
)
def bpasscal2(msfile, params, field=None):
refant = params['flagging']['refant']
bpassgaintable = params['calibration']['bpassgaintable']
bpasstable = params['calibration']['bpasstable']
gaintables = params['calibration']['gaintables']
uvran = params['general']['uvran']
ch0_spw = params['calibration']['ch0_spw']
ini_fgloops = params['calibration']['ini_nloops']
fluxcal = params['general']['fluxcal']
# Initial gain calibration
# for i in range(ini_fgloops):
print('Running the initial gaincal pre-bandpass ...')
cts.gaincal(msfile,
caltable=bpassgaintable,
field=field,
uvrange=uvran,
spw=ch0_spw,
solint='120s',
solmode='R',
refant=refant,
minsnr=2.0,
gaintype='G',
calmode='p',
gaintable=gaintables,
interp=[
'linear, linear',
],
append=True,
parang=False)
'''
for j in fluxcal:
#print('Applying the calibration table to ', i)
cts.applycal(msfile, field=j, spw=ch0_spw, gaintable=bpassgaintable, gainfield=[
'', '', j], uvrange=uvran, interp=['', '', 'linear'], calwt=[False], parang=False)
#cts.flagdata(msfile, mode='tfcrop', datacolumn='corrected', field=j, ntime='scan', spw=ch0_spw,
# timecutoff=6, freqcutoff=6, timefit='poly',freqfit='line',flagdimension='time',
# extendflags=False, timedevscale=5.0,freqdevscale=5.0, extendpols=False,growaround=False,
# action='apply', flagbackup=True,overwrite=True, writeflags=True)
#cts.flagdata(msfile, params, field=j, tcut=6, fcut=6, instance='postcal')
#flg.extend(msfile, params, field=j, grow=80, instance='postcal')
cts.clearcal(msfile) # Removes the corrected datacolumn
if i == (ini_fgloops - 1):
print('Single channel flagcal loops ended.')
break
else:
subprocess.run('rm -r {}'.format(bpassgaintable), shell=True, check=True)
'''
gaintables.append(bpassgaintable)
print(gaintables, 'before bpass')
# bpass calibration
cts.bandpass(msfile,
caltable=bpasstable,
spw='0',
field=field,
uvrange=uvran,
solint='inf',
refant=refant,
solnorm=True,
minsnr=2.0,
append=True,
fillgaps=8,
parang=False,
gaintable=gaintables,
interp=['linear,linear', 'linear,linear'])
#cts.bandpass(msfile, caltable=bpasstable, spw ='0', field=field, solint='inf', refant=refant, solnorm = True, minsnr=2.0, fillgaps=8, bandtype='BPOLY', degamp=5, degphase=7, parang = True, gaintable=gaintables, interp=['nearest,nearestflag','nearest,nearestflag'])
# Removing and also deleting the on-the fly gaincal table used for bandpass
subprocess.run('rm -r {}'.format(bpassgaintable), shell=True, check=True)
gaintables.remove(bpassgaintable)
return print('Bandpass calibration done')
# Display the output restored image
dispImage('try3.image')
# In[497]:
# Display the new residual image
dispImage('try3.residual')
# This image shows artifacts from gain errors (different from the pure noise-like errors in the previous simulation)
#
# Calculate gain solutions (since we have already saved the model)
# In[498]:
bandpass(vis='sim_data.ms', caltable='sc.tab', solint='int')
# Apply gain solutions
# In[499]:
applycal(vis='sim_data.ms', gaintable='sc.tab')
# In[500]:
## Plot Calibrated data
plotData(myplot='corr_spectrum')
# Compare with the above uncalibrated data. Also, compare with visibilities simulated just with noise. Subsequent imaging should use the corrected_data column.
# In[501]: