def calibration_plots(primary: str, secondary: str, freq: str):
"""Common function to create calibration plots of the primary and
secondary visibility files
Arguments:
primary {str} -- Filename of primary calibrator
secondary {str} -- Filename of the secondary calibrator
freq {str} -- Frequency of the IF
"""
plt = [
m(f'uvplt vis={primary} axis=time,amp options=nob,nof stokes=i device=primary_timeamp_{freq}.png/PNG'
),
m(f'uvplt vis={primary} axis=re,im options=nob,nof,eq stokes=i,q,u,v device=primary_reim_{freq}.png/PNG'
),
m(f'uvplt vis={primary} axis=uc,vc options=nob,nof stokes=i device=primary_ucvc_{freq}.png/PNG'
),
m(f'uvplt vis={primary} axis=FREQ,amp options=nob,nof stokes=i device=primary_freqamp_{freq}.png/PNG'
),
m(f'uvplt vis={secondary} axis=time,amp options=nob,nof stokes=i device=secondary_timeamp_{freq}.png/PNG'
),
m(f'uvplt vis={secondary} axis=re,im options=nob,nof,eq stokes=i,q,u,v device=secondary_reim_{freq}.png/PNG'
),
m(f'uvplt vis={secondary} axis=uc,vc options=nob,nof stokes=i device=secondary_ucvc_{freq}.png/PNG'
),
m(f'uvplt vis={secondary} axis=FREQ,amp options=nob,nof stokes=i device=secondary_freqamp_{freq}.png/PNG'
),
m(f'uvfmeas vis={secondary} stokes=i log=secondary_uvfmeas_{freq}_log.txt device=secondary_uvfmeas_{freq}.png/PNG'
)
]
pool = Pool(7)
result = pool.map(run, plt)
pool.close()
pool.join()
def mosaic_pgflag(src):
"""The name of the C3171 mosaic file to flag
Arguments:
src {str} -- Name of the file to flag
"""
pgflag = m(f"pgflag vis={src} command='<b' stokes=i,v,q,u flagpar=8,2,2,3,6,3 "\
f"options=nodisp").run()
logger.log(logging.INFO, pgflag)
pgflag = m(f"pgflag vis={src} command='<b' stokes=i,v,u,q flagpar=8,2,2,3,6,3 "\
f"options=nodisp").run()
logger.log(logging.INFO, pgflag)
def pgflag_flag(s):
"""Flag uvdata based on the smonrms statistic
s {str} -- Path to the filename to flag
"""
# Automated flagging
pgflag = m(f"pgflag vis={s} command='<b' stokes=i,q,u,v flagpar=8,5,0,3,6,3 options=nodisp").run()
print(pgflag)
pgflag = m(f"pgflag vis={s} command='<b' stokes=i,v,q,u flagpar=8,2,0,3,6,3 options=nodisp").run()
print(pgflag)
pgflag = m(f"pgflag vis={s} command='<b' stokes=i,v,u,q flagpar=8,2,0,3,6,3 options=nodisp").run()
print(pgflag)
def resolution(self):
"""Fit to the restor image to return the image resolutions
"""
if 'restor' not in self.img_tasks.keys():
return None
invert = self.img_tasks['invert']
imfit = m(f"imfit in={invert.beam} object=beam region='perc(1)(1)'").run()
print(imfit)
src_data = {}
obj1 = False
for line in imfit.p.stdout.split('\n'):
if 'Object type: beam' in line:
obj1 = True
continue
elif not obj1:
continue
if 'Major axis' in line:
items = line.split()
src_data['bmaj'] = float(items[-3])
src_data['bmaj_e'] = float(items[-1])
elif 'Minor axis' in line:
items = line.split()
src_data['bmin'] = float(items[-3])
src_data['bmin_e'] = float(items[-1])
elif 'Position angle' in line:
items = line.split()
src_data['pa'] = float(items[-3])
src_data['pa_e'] = float(items[-1])
return src_data
def mosaic_src_calibration(src: str):
"""Apply any common calibration steps for each source file
Arguments:
src {str} -- uv source file of item to process
"""
gpaver = m(f"gpaver vis={src} interval=5 options=scalar").run()
logger.log(logging.INFO, gpaver)
def calibrator_pgflag(src):
"""A series of pgflag steps common to most (if not all) of
the primary and secondary miriad uv files.
This is following the flagging outlined in Do_Flag.csh` on the
ATCAGAMA wiki
Arguments:
src {str} -- The filename of the data to flag
"""
primary = True if '1934-638' in src else False
if primary:
pgflag = m(
f"pgflag vis={src} stokes=v flagpar=15,3,3,3,5,3 command='<' options=nodisp"
).run()
logger.log(logging.INFO, pgflag)
pgflag = m(
f"pgflag vis={src} stokes=q flagpar=15,3,3,3,5,3 command='<' options=nodisp"
).run()
logger.log(logging.INFO, pgflag)
pgflag = m(
f"pgflag vis={src} stokes=u flagpar=15,3,3,3,5,3 command='<' options=nodisp"
).run()
logger.log(logging.INFO, pgflag)
pgflag = m(
f"pgflag vis={src} stokes=i flagpar=15,3,3,3,5,3 command='<' options=nodisp"
).run()
logger.log(logging.INFO, pgflag)
else:
pgflag = m(
f"pgflag vis={src} stokes=i,q,u,v flagpar=10,1,1,3,5,3 command='<' options=nodisp"
).run()
logger.log(logging.INFO, pgflag)
pgflag = m(
f"pgflag vis={src} stokes=i,u flagpar=10,1,1,3,5,3 command='<' options=nodisp"
).run()
logger.log(logging.INFO, pgflag)
pgflag = m(
f"pgflag vis={src} stokes=i,q flagpar=10,1,1,3,5,3 command='<' options=nodisp"
).run()
logger.log(logging.INFO, pgflag)
pgflag = m(
f"pgflag vis={src} stokes=i flagpar=10,1,1,3,5,3 command='<' options=nodisp"
).run()
logger.log(logging.INFO, pgflag)
def calibrator_pgflag(src):
"""A series of pgflag steps common to most (if not all) of
the primary and secondary miriad uv files
Arguments:
src {str} -- The filename of the data to flag
"""
# Automated flagging
pgflag = m(
f"pgflag vis={src} command='<b' stokes=i,q,u,v flagpar=8,5,5,3,6,3 options=nodisp"
).run()
logger.log(logging.INFO, pgflag)
pgflag = m(
f"pgflag vis={src} command='<b' stokes=i,v,q,u flagpar=8,2,2,3,6,3 options=nodisp"
).run()
logger.log(logging.INFO, pgflag)
pgflag = m(
f"pgflag vis={src} command='<b' stokes=i,v,u,q flagpar=8,2,2,3,6,3 options=nodisp"
).run()
logger.log(logging.INFO, pgflag)
def selfcal(self, *args, round: int=0, self_kwargs: dict=None, preprocess=None, postprocess=None,
**kwargs):
"""Apply a round of selfcalibration to the uv-file if appropriate.
Keyword Arguments:
round {int} -- The selfcalibration round the process is up to (default: {0})
self_kwargs {dict} -- Options for selfcalibration. Can be overwritten by attempt_selfcal returns (default: {None})
preprocess {callable} -- callable to accept an instance of uv-file before selfcalibration (default: {None})
postprocess {callable} -- callable to accept an instance of uv-file after selfcalibration (default: {None})
"""
run_self = self.attempt_selfcal(**kwargs)
if isinstance(run_self, tuple):
run_self, self_kwargs = run_self
# Conditions are not met. Return instance of self.
if not run_self:
return self
# Apply calibration tables in existing uv-file
uvaver = m(f"uvaver vis={self.uv} out={self.uv}.{round}").run()
print(uvaver)
# Action any described todos
if preprocess is not None:
preprocess(uvaver.out)
# Derive selfcalibrated solutions
selfcal = m(f"selfcal vis={uvaver.out} model={self.img_tasks['mfclean'].out} "\
f"options=mfs,phase interval=0.1 nfbin=4", over=self_kwargs).run()
print(selfcal)
# Action any described todos
if postprocess is not None:
postprocess(uvaver.out)
# Return new instance to selfcaled file
return uv(uvaver.out)
def flag_inttime(uv: str,
threshold: int = 9,
logfile: str = 'dump.txt',
selection: str = 'flagging.txt',
delete: bool = False):
"""Flag out short integration cycles from a uv-file
Arguments:
uv {str} -- uv-file to flag
Keyword Arguments:
threshold {int} -- Minimum length of time for each cycle (default: {9})
logfile {str} -- File to `uvdump` to (default: {'dump.txt'})
selection {str} -- File to place select statements in (default: {'flagging.txt'})
delete {bool} -- Clean up log files (default: {False})
"""
uvdump = m(
f"uvdump vis={uv} vars=inttime,ant1,ant2,time,source log={logfile}"
).run()
print(uvdump)
var = pd.read_csv(logfile,
names=('inttime', 'ant1', 'ant2', 'time', 'source'))
var = var[(var['ant1'].values == 1) &
(var['ant2'].values == 2)] # Smaller list of selects
var[var['inttime'] < threshold].apply(flag_cycle,
axis=1,
selection=selection)
uvflag = m(f"uvflag vis={uv} select=@{selection} flagval=flag").run()
print(uvflag)
if delete:
os.remove(logfile)
os.remove(selection)
def mosaic_uvsplit(mosaic: str):
"""uvsplit a mosaic source file up based on source names. Will return
a list of new source files to operate upon.
Arguments:
mosaic {str} -- Name of the mosaic file to split
"""
uvsplit = m(f"uvsplit vis={mosaic}").run()
logger.log(logging.INFO, uvsplit)
srcs = []
for line in str(uvsplit).splitlines():
if 'Creating' not in line:
continue
srcs.append(line.split()[1])
return srcs
def sc_round_4(s):
"""Logic for the four round of selfcalibration
Arguments:
s {uv} -- An instance of the uv-class.
"""
from astropy.io import fits as pyfits
restor = s.img_tasks['restor']
fits = m(f"fits in={restor.out} out={restor.out}.4.fits op=xyout " \
"region='images(1,1)'").run()
data = pyfits.open(fits.out)[0].data.squeeze()
delete_miriad(fits.out)
# Threshold selected by experimentation (does it look bad?)
restor_max = data.max()
if restor_max > 400*s.img_tasks['stokes_v_rms']:
return True, {'options':'mfs,amp', 'interval':'1'}
else:
return False
def sc_round_2(s):
"""Logic for the second round of selfcalibration
Arguments:
s {uv} -- An instance of the uv-class.
"""
from astropy.io import fits as pyfits
restor = s.img_tasks['restor']
fits = m(f"fits in={restor.out} out={restor.out}.2.fits op=xyout " \
"region='images(1,1)'").run()
data = pyfits.open(fits.out)[0].data.squeeze()
delete_miriad(fits.out)
# Threshold selected by dumb luck and subsequet experimentation
restor_max = data.max()
if restor_max > 50*s.img_tasks['stokes_v_rms']:
return True, {'interval':'1'}
else:
return False
def mosaic_src_pgflag(src):
"""Thw flagging procedure applied to the source data. THis follows Minh's script
called Do_Flag.csh on the ATCAGAMA wiki. For ease it is applied before uvsplit.
Time convolution is turned off for the moment.
Arguments:
src {[type]} -- [description]
"""
pgflag = m(f"pgflag vis={src} command='<' stokes=i,q,u,v flagpar=10,1,0,3,5,3 "\
f"options=nodisp").run()
logger.log(logging.INFO, pgflag)
pgflag = m(f"pgflag vis={src} command='<' stokes=i,q,u,v flagpar=10,1,0,3,5,3 "\
f"options=nodisp").run()
logger.log(logging.INFO, pgflag)
pgflag = m(f"pgflag vis={src} command='<' stokes=i,q flagpar=10,1,0,3,5,3 "\
f"options=nodisp").run()
logger.log(logging.INFO, pgflag)
pgflag = m(f"pgflag vis={src} command='<' stokes=i,q flagpar=10,1,0,3,5,3 "\
f"options=nodisp").run()
logger.log(logging.INFO, pgflag)
pgflag = m(f"pgflag vis={src} command='<' stokes=i,u flagpar=10,1,0,3,5,3 "\
f"options=nodisp").run()
logger.log(logging.INFO, pgflag)
pgflag = m(f"pgflag vis={src} command='<' stokes=i,u flagpar=10,1,0,3,5,3 "\
f"options=nodisp").run()
logger.log(logging.INFO, pgflag)
pgflag = m(f"pgflag vis={src} command='<' stokes=i flagpar=10,1,0,3,5,3 "\
f"options=nodisp").run()
logger.log(logging.INFO, pgflag)
pgflag = m(f"pgflag vis={src} command='<' stokes=i flagpar=10,1,0,3,5,3 "\
f"options=nodisp").run()
logger.log(logging.INFO, pgflag)
def run_linmos(mos: list, round: int=0, mode: str='restor'):
"""The the miriad task linmos agaisnt list of uv files
Arguments:
mos {list} -- A list of uv-instances
round {int} -- A int for self calibration round
mode {str} -- The mode to linmos together
"""
convol_files = f'temp_file_{round}.{mode}.dat'
with open(f"{convol_files}", 'w') as f:
for i in mos:
print(f"{i.img_tasks[f'convol_{mode}'].out}", file=f)
out = f"all_days.{round}.{mode}.linmos"
delete_miriad(out)
linmos = m(f"linmos in=@{convol_files} bw=4.096 out={out}").run()
print(linmos)
os.remove(convol_files)
return linmos
def convol(self, bmaj: float, bmin: float, pa: float, mode: str='restor'):
"""Convol a restored image to a different resolution
Arguments:
bmaj {float} -- Beam major in arcseconds
bmin {float} -- Beam minor in arcseconds
pa {float} -- Beam position angle in degrees
mode {str} -- The mode to convol (default: {restor})
"""
if mode not in self.img_tasks.keys() or \
f'convol_{mode}' in self.img_tasks.keys():
return None
restor = self.img_tasks[mode]
print(restor.out)
convol = m(f"convol map={restor.out} out={self.uv}.{mode}.convol fwhm={bmaj},{bmin} "\
f"pa={pa} options=final").run()
print(convol)
self.img_tasks[f'convol_{mode}'] = convol
def image(self, invert_kwargs: dict=None, cut_thres_1: {float}=5, cut_thres_2: {float}=None):
"""A simple MFCLEAN imaging pipeline
Keyword Arguments:
invert_kwargs {dict} -- Options to pass to the invert map/beam stage (default: {None})
cut_thres_1 {float} -- First cutoff option provided to mfclean, depth to clean to (default: {5})
invert_kwargs {dict} -- Second option provided to mfclean if not None, depth to clean existing components down to (default: {None})
"""
# No work to do, as data has been imaged
if 'restor' in self.img_tasks.keys():
return
invert = m(f"invert vis={self.uv} options=mfs,sdb,double,mosaic,systemp " \
f"offset=3:32:22.0,-27:48:37 stokes=i imsize=5,5,beam " \
f"map={self.uv}.map beam={self.uv}.beam robust=1 cell=0.25",
over=invert_kwargs).run()
print(invert)
stokes_v = m(f"invert vis={self.uv} imsize=3,3,beam options=mfs,sdb,double,mosaic,systemp " \
f"offset=3:32:22.0,-27:48:37 stokes=v " \
f"map={self.uv}.v.map cell=0.35").run()
print(stokes_v)
sigest = m(f"sigest in={stokes_v.map}").run()
print(sigest)
for l in sigest.p.stdout.split('\n'):
if 'Estimated rms' in l:
v_rms = float(l.split()[-1])
cutoff = f"{cut_thres_1*v_rms},{cut_thres_2*v_rms}" if cut_thres_2 is not None else f"{cut_thres_1*v_rms}"
mfclean = m(f"mfclean map={invert.map} beam={invert.beam} out={self.uv}.clean "\
f"region='perc(66)' niters=1500 cutoff={cutoff}").run()
print(mfclean)
restor = m(f"restor map={invert.map} beam={invert.beam} model={mfclean.out} "\
f"options=mfs out={self.uv}.restor").run()
print(restor)
residual = m(f"restor map={invert.map} beam={invert.beam} model={mfclean.out} "\
f"mode=residual out={self.uv}.residual").run()
print(residual)
self.img_tasks = {'invert':invert, 'invert_v':stokes_v, 'mfclean':mfclean,
'restor':restor, 'residual':residual,
'stokes_v_rms':v_rms}
def run(a):
a.run()
logger.log(logging.INFO, a)
# Initial pass of the data loads in without the Tsys correction to produce a
# robust spectrum of the secondary calibrator
for index, freq in enumerate(FREQS):
ifsel = index + 1
primary = f"{mu.primary}.{freq}"
secondary = f"{mu.secondary}.{freq}"
mosaic = f"{mu.science}.{freq}"
# Load in the data
atlod = m(f"atlod in=raw/*C3171 options=notsys,rfiflag,birdie,xycorr,noauto "\
f"ifsel={ifsel} out=data{ifsel}.uv").run()
logger.log(logging.INFO, atlod)
# Flag the known bad channels out
if ifsel == 1:
mu.uvflag(atlod.out, mu.flags_7)
else:
mu.uvflag(atlod.out, mu.flags_9)
# Split the data up
uvsplit = m(f"uvsplit vis={atlod.out} options=mosaic "\
f"select=source({mu.primary}),source({mu.secondary})").run()
logger.log(logging.INFO, uvsplit)
mfcal = m(f"mfcal vis={primary} interval=0.1,0.1,5").run()
gpcal = m(f"gpcal vis={primary} interval=0.1 nfbin={NFBIN} "\
NFBIN = 4
FREQ = 9500
# Load in files assuming the setup file/s have been renamed or deleted
# Can lead to problems with 0 and 9s.
files = glob('raw/*C3132')
# Example loading in files assuming first is setup
# files = glob('raw/*C3132').pop(0)
# Glob order is not the same as sort order
files = sorted(files)
# Load in data. Remember to set ifsel appropriately
atlod = m(
f"atlod in={','.join(files)} out=data9.uv ifsel=2 options=birdie,rfiflag,noauto,xycorr"
).run()
logger.log(logging.INFO, atlod)
# Flag out known bad channels
mu.uvflag(atlod.out, mu.flags_9)
uvsplit = m(f"uvsplit vis={atlod.out} options=mosaic").run()
logger.log(logging.INFO, uvsplit)
# Deduce the objects in the observing run
srcs = mu.derive_obs_sources(uvsplit, FREQ)
primary, secondary, mosaic_targets = srcs
mu.calibrator_pgflag(primary)
def seeing_flag(s):
"""Flag uvdata based on the smonrms statistic
"""
uvflag = m(f"uvflag vis={s} select=-seeing(500) flagval=flag").run()
print(uvflag)
def attempt_selfcal(self, mode: str=None):
"""Logic to decide whether selcalibration should be attempted. Mode can be
a function that is passed into the method. It should except as a single
argument a reference to uv-instance.
Keyword Arguments:
mode {str} -- The mode used to evaluate decision to self (default: {None})
Returns:
bool -- Whether selfcal should be used
dict -- optional return with selfcal key/values to use
"""
# Nothing set
if mode is None:
return False
# If it quacks like a duck
try:
return mode(self)
except Exception as e:
print(e)
pass
if mode == 'test':
# Example of conditional. Used to test subequent imaging
if '100' in self.uv:
return False
elif mode == 'restor_max':
from astropy.io import fits as pyfits
restor = self.img_tasks['restor']
fits = m(f"fits in={restor.out} out={restor.out}.fits op=xyout " \
"region='images(1,1)'").run()
data = pyfits.open(fits.out)[0].data.squeeze()
delete_miriad(fits.out)
# Threshold selected by dumb luck and subsequet experimentation
restor_max = data.max()
if restor_max > 150*self.img_tasks['stokes_v_rms']:
return True, {'options':'mfs,amp'}
elif restor_max > 50*self.img_tasks['stokes_v_rms']:
return True
else:
return False
elif mode == 'clean_sum':
from astropy.io import fits as pyfits
clean = self.img_tasks['mfclean']
fits = m(f"fits in={clean.out} out={clean.out}.fits op=xyout " \
"region='images(1,1)'").run()
data = pyfits.open(fits.out)[0].data.squeeze()
delete_miriad(fits.out)
clean_sum = data.sum()
if clean_sum > 500*self.img_tasks['stokes_v_rms']:
return True, {'options':'mfs,amp'}
elif clean_sum > 100*self.img_tasks['stokes_v_rms']:
return True
else:
return False
return True
