def getPlotantsObservatoryInfo(msname):
metadata = msmetadata()
metadata.open(msname)
telescope = metadata.observatorynames()[0]
arrayPos = metadata.observatoryposition()
metadata.close()
return telescope, arrayPos
def polfield_name(visname):
from casatools import msmetadata
msmd = msmetadata()
msmd.open(visname)
fieldnames = msmd.fieldnames()
msmd.done()
polfield = ''
if any([
ff in ["3C286", "1328+307", "1331+305", "J1331+3030"]
for ff in fieldnames
]):
polfield = list(
set(["3C286", "1328+307", "1331+305",
"J1331+3030"]).intersection(set(fieldnames)))[0]
elif any([
ff in ["3C138", "0518+165", "0521+166", "J0521+1638"]
for ff in fieldnames
]):
polfield = list(
set(["3C138", "0518+165", "0521+166",
"J0521+1638"]).intersection(set(fieldnames)))[0]
elif any([
ff in ["3C48", "0134+329", "0137+331", "J0137+3309"]
for ff in fieldnames
]):
polfield = list(
set(["3C48", "0134+329", "0137+331",
"J0137+3309"]).intersection(set(fieldnames)))[0]
else:
logger.warning(
"No valid polarization field found. Defaulting to use the phase calibrator to solve for XY phase."
)
logger.warning(
"The polarization solutions found will likely be wrong. Please check the results carefully."
)
return polfield
import matplotlib
# Agg doesn't need X - matplotlib doesn't work with xvfb
matplotlib.use('Agg', warn=False)
import matplotlib.pyplot as plt
import numpy as np
from config_parser import validate_args as va
import bookkeeping
import glob
PLOT_DIR = 'plots'
EXTN = 'png'
from casatasks import *
from casatools import table,msmetadata
tb = table()
msmd = msmetadata()
logfile=casalog.logfile()
casalog.setlogfile('logs/{SLURM_JOB_NAME}-{SLURM_JOB_ID}.casa'.format(**os.environ))
import logging
from time import gmtime
logging.Formatter.converter = gmtime
logger = logging.getLogger(__name__)
logging.basicConfig(format="%(asctime)-15s %(levelname)s: %(message)s", level=logging.INFO)
def avg_ants(arrlist):
return [np.mean(arr, axis=-1) for arr in arrlist]
def lengthen(edat, inpdat):
"""
def get_selfcal_args(vis, loop, nloops, nterms, deconvolver, discard_nloops,
calmode, outlier_threshold, threshold, step):
visbase = os.path.split(
vis.rstrip('/ '))[1] # Get only vis name, not entire path
#Assume first target will be in the visname later (relevant for writing outliers.txt at beginning of pipeline)
if '.ms' in visbase:
from casatools import msmetadata
msmd = msmetadata()
msmd.open(vis)
basename = visbase.replace(
'.ms', '.{0}'.format(
msmd.namesforfields(msmd.fieldsforintent('TARGET'))[0]))
msmd.done()
else:
basename = visbase.replace('.mms', '')
imbase = basename + '_im_%d' # Images will be produced in $CWD
imagename = imbase % loop
outimage = imagename + '.image'
pixmask = imagename + ".pixmask"
maskfile = imagename + ".islmask"
rmsfile = imagename + ".rms"
caltable = basename + '.gcal%d' % loop
prev_caltables = sorted(glob.glob('*.gcal?'))
cfcache = basename + '.cf'
thresh = 10
if nterms[loop] > 1 and deconvolver[loop] == 'mtmfs':
outimage += '.tt0'
if step not in ['tclean', 'sky'] and not os.path.exists(outimage):
logger.error(
"Image '{0}' doesn't exist, so self-calibration loop {1} failed. Will terminate selfcal process."
.format(outimage, loop))
sys.exit(1)
if step in ['tclean', 'predict']:
pixmask = imbase % (loop - 1) + '.pixmask'
rmsfile = imbase % (loop - 1) + '.rms'
if step in ['tclean', 'predict', 'sky'
] and ((loop == 0 and not os.path.exists(pixmask)) or
(0 < loop < nloops and calmode[loop] == '')):
pixmask = ''
#Check no missing caltables
for i in range(0, loop):
if calmode[i] != '' and not os.path.exists(basename + '.gcal%d' % i):
logger.error(
"Calibration table '{0}' doesn't exist, so self-calibration loop {1} failed. Will terminate selfcal process."
.format(basename + '.gcal%d' % i, i))
sys.exit(1)
for i in range(discard_nloops):
prev_caltables.pop(0)
if outlier_threshold != '' and outlier_threshold != 0: # and (loop > 0 or step in ['sky','bdsf'] and loop == 0):
if step in ['tclean', 'predict', 'sky']:
outlierfile = 'outliers_loop{0}.txt'.format(loop)
else:
outlierfile = 'outliers_loop{0}.txt'.format(loop + 1)
else:
outlierfile = ''
if not (type(threshold[loop]) is str
and 'Jy' in threshold[loop]) and threshold[loop] > 1:
if step in ['tclean', 'predict']:
if os.path.exists(rmsfile):
from casatasks import imstat
stats = imstat(imagename=rmsfile)
threshold[loop] *= stats['min'][0]
else:
logger.error(
"'{0}' doesn't exist. Can't do thresholding at S/N > {1}. Loop 0 must use an absolute threshold value. Check the logs to see why RMS map not created."
.format(rmsfile, threshold[loop]))
sys.exit(1)
elif step == 'bdsf':
thresh = threshold[loop]
return imbase, imagename, outimage, pixmask, rmsfile, caltable, prev_caltables, threshold, outlierfile, cfcache, thresh, maskfile
def quicklook_line_imaging(
myvis,
thisgal,
linespw_dict,
nchan_vel=5,
# channel_width_kms=20.,
niter=0,
nsigma=5.,
imsize_max=800,
overwrite_imaging=False,
export_fits=True):
if not os.path.exists("quicklook_imaging"):
os.mkdir("quicklook_imaging")
this_vsys = target_vsys_kms[thisgal]
# Pick our line range based on the HI for all lines.
this_velrange = target_line_range_kms[thisgal]['HI']
# We have a MW foreground window on some targets. Skip this for the galaxy range.
if isinstance(this_velrange[0], list):
for this_range in this_velrange:
if min(this_range) < this_vsys < max(this_range):
this_velrange = this_range
break
# Check that the search for the right velocity range didn't fail
if isinstance(this_velrange[0], list):
raise ValueError(
f"Unable to find range with target vsys ({this_vsys}) from {this_velrange}."
f" Check the velocity ranges defined in target_setup.py for {thisgal}"
)
# width_vel = channel_width_kms
# width_vel_str = f"{width_vel}km/s"
start_vel = f"{int(min(this_velrange))}km/s"
# nchan_vel = int(abs(this_velrange[0] - this_velrange[1]) / width_vel)
width_vel = int(
round(abs(this_velrange[0] - this_velrange[1]) / float(nchan_vel)))
width_vel_str = f"{width_vel}km/s"
# Select only the non-continuum SPWs
line_spws = []
for thisspw in linespw_dict:
if "continuum" not in linespw_dict[thisspw]['label']:
# Our 20A-346 tracks have a combined OH1665/1667 SPW. Split into separate cubes in this case
line_labels = linespw_dict[thisspw]['label'].split("-")
for line_label in line_labels:
line_spws.append([str(thisspw), line_label])
# Select our target fields. We will loop through
# to avoid the time + memory needed for mosaics.
synthutil = synthesisutils()
myms = ms()
# if no fields are provided use observe_target intent
# I saw once a calibrator also has this intent so check carefully
# mymsmd.open(vis)
myms.open(myvis)
mymsmd = myms.metadata()
target_fields = mymsmd.fieldsforintent("*TARGET*", True)
mymsmd.close()
myms.close()
t0 = datetime.datetime.now()
# Loop through targets and line SPWs
for target_field in target_fields:
casalog.post(f"Quick look imaging of field {target_field}")
# Loop through the SPWs to identify the biggest image size needed.
# For ease downstream, we will use the same imsize for all SPWs.
# NOTE: for L-band, that's a factor of ~2 difference. It may be more pronounced in other
# bands
cell_size = {}
imsizes = []
for thisspw_info in line_spws:
thisspw, line_name = thisspw_info
# Ask for cellsize
this_im = imager()
this_im.selectvis(vis=myvis, field=target_field, spw=str(thisspw))
image_settings = this_im.advise()
this_im.close()
# When all data is flagged, uvmax = 0 so cellsize = 0.
# Check for that case to avoid tclean failures
# if image_settings[2]['value'] == 0.:
# casalog.post(f"All data flagged for {this_imagename}. Skipping")
# continue
# NOTE: Rounding will only be reasonable for arcsec units with our L-band setup.
# Could easily fail on ~<0.1 arcsec cell sizes.
cell_size[thisspw] = [
image_settings[2]['value'], image_settings[2]['unit']
]
# No point in estimating image size for an empty SPW.
if image_settings[2]['value'] == 0.:
continue
# For the image size, we will do an approx scaling was
# theta_PB = 45 / nu (arcmin)
this_msmd = msmetadata()
this_msmd.open(myvis)
mean_freq = this_msmd.chanfreqs(
int(thisspw)).mean() / 1.e9 # Hz to GHz
this_msmd.close()
approx_pbsize = 1.2 * (45. / mean_freq) * 60 # arcsec
approx_imsize = synthutil.getOptimumSize(
int(approx_pbsize / image_settings[2]['value']))
imsizes.append(approx_imsize)
if len(imsizes) == 0:
casalog.post(f"{target_field} is fully flagged. Skipping.")
continue
this_imsize = min(imsize_max, max(imsizes))
for thisspw_info in line_spws:
thisspw, line_name = thisspw_info
casalog.post(
f"Quick look imaging of field {target_field} SPW {thisspw}")
target_field_label = target_field.replace('-', '_')
this_imagename = f"quicklook_imaging/quicklook-{target_field_label}-spw{thisspw}-{line_name}-{myvis}"
if export_fits:
check_exists = os.path.exists(f"{this_imagename}.image.fits")
else:
check_exists = os.path.exists(f"{this_imagename}.image")
if check_exists:
if overwrite_imaging:
rmtables(f"{this_imagename}*")
os.remove(f"{this_imagename}.image.fits")
else:
casalog.post(f"Found {this_imagename}. Skipping imaging.")
continue
if cell_size[thisspw][0] == 0:
casalog.post(
f"All data flagged for {this_imagename}. Skipping")
continue
this_cellsize = f"{round(cell_size[thisspw][0] * 0.8, 1)}{cell_size[thisspw][1]}"
this_pblim = 0.5
this_nsigma = nsigma
this_niter = niter
# Clean up any possible imaging remnants first
rmtables(f"{this_imagename}*")
tclean(vis=myvis,
field=target_field,
spw=str(thisspw),
cell=this_cellsize,
imsize=this_imsize,
specmode='cube',
weighting='briggs',
robust=0.0,
start=start_vel,
width=width_vel_str,
nchan=nchan_vel,
niter=this_niter,
nsigma=this_nsigma,
imagename=this_imagename,
restfreq=f"{linerest_dict_GHz[line_name]}GHz",
pblimit=this_pblim)
if export_fits:
exportfits(imagename=f"{this_imagename}.image",
fitsimage=f"{this_imagename}.image.fits",
history=False,
overwrite=True)
# Clean-up extra imaging products if they are not needed.
cleanup_misc_quicklook(this_imagename,
remove_psf=True,
remove_residual=this_niter == 0,
remove_image=True if export_fits else False)
t1 = datetime.datetime.now()
casalog.post(f"Quicklook line imaging took {t1 - t0}")
def quicklook_continuum_imaging(myvis,
contspw_dict,
niter=0,
nsigma=5.,
imsize_max=800,
overwrite_imaging=False,
export_fits=True):
'''
Per-SPW MFS, nterm=1, dirty images of the targets
'''
if not os.path.exists("quicklook_imaging"):
os.mkdir("quicklook_imaging")
# Select only the continuum SPWs (in case there are any line SPWs).
continuum_spws = []
for thisspw in contspw_dict:
if "continuum" in contspw_dict[thisspw]['label']:
continuum_spws.append(str(thisspw))
# Select our target fields. We will loop through
# to avoid the time + memory needed for mosaics.
synthutil = synthesisutils()
myms = ms()
# if no fields are provided use observe_target intent
# I saw once a calibrator also has this intent so check carefully
# mymsmd.open(vis)
myms.open(myvis)
mymsmd = myms.metadata()
target_fields = mymsmd.fieldsforintent("*TARGET*", True)
mymsmd.close()
myms.close()
t0 = datetime.datetime.now()
# Loop through targets and line SPWs
for target_field in target_fields:
casalog.post(f"Quick look imaging of field {target_field}")
cell_size = {}
imsizes = []
for thisspw in continuum_spws:
# Ask for cellsize
this_im = imager()
this_im.selectvis(vis=myvis, field=target_field, spw=str(thisspw))
image_settings = this_im.advise()
this_im.close()
# When all data is flagged, uvmax = 0 so cellsize = 0.
# Check for that case to avoid tclean failures
# if image_settings[2]['value'] == 0.:
# casalog.post(f"All data flagged for {this_imagename}. Skipping")
# continue
# NOTE: Rounding will only be reasonable for arcsec units with our L-band setup.
# Could easily fail on ~<0.1 arcsec cell sizes.
cell_size[thisspw] = [
image_settings[2]['value'], image_settings[2]['unit']
]
# No point in estimating image size for an empty SPW.
if image_settings[2]['value'] == 0.:
continue
# For the image size, we will do an approx scaling was
# theta_PB = 45 / nu (arcmin)
this_msmd = msmetadata()
this_msmd.open(myvis)
mean_freq = this_msmd.chanfreqs(
int(thisspw)).mean() / 1.e9 # Hz to GHz
this_msmd.close()
approx_pbsize = 1.2 * (45. / mean_freq) * 60 # arcsec
approx_imsize = synthutil.getOptimumSize(
int(approx_pbsize / image_settings[2]['value']))
imsizes.append(approx_imsize)
if len(imsizes) == 0:
casalog.post(f"{target_field} is fully flagged. Skipping.")
continue
this_imsize = min(imsize_max, max(imsizes))
for thisspw in continuum_spws:
casalog.post(
f"Quick look imaging of field {target_field} SPW {thisspw}")
target_field_label = target_field.replace('-', '_')
this_imagename = f"quicklook_imaging/quicklook-{target_field_label}-spw{thisspw}-continuum-{myvis}"
if export_fits:
check_exists = os.path.exists(f"{this_imagename}.image.fits")
else:
check_exists = os.path.exists(f"{this_imagename}.image")
if check_exists:
if overwrite_imaging:
rmtables(f"{this_imagename}*")
os.remove(f"{this_imagename}.image.fits")
else:
casalog.post(f"Found {this_imagename}. Skipping imaging.")
continue
if cell_size[thisspw][0] == 0:
casalog.post(
f"All data flagged for {this_imagename}. Skipping")
continue
this_cellsize = f"{round(cell_size[thisspw][0] * 0.8, 1)}{cell_size[thisspw][1]}"
this_pblim = 0.5
this_nsigma = nsigma
this_niter = niter
# Clean up any possible imaging remnants first
rmtables(f"{this_imagename}*")
tclean(vis=myvis,
field=target_field,
spw=str(thisspw),
cell=this_cellsize,
imsize=this_imsize,
specmode='mfs',
nterms=1,
weighting='briggs',
robust=0.0,
niter=this_niter,
nsigma=this_nsigma,
fastnoise=True,
imagename=this_imagename,
pblimit=this_pblim)
if export_fits:
exportfits(imagename=f"{this_imagename}.image",
fitsimage=f"{this_imagename}.image.fits",
history=False,
overwrite=True)
# Clean-up extra imaging products if they are not needed.
cleanup_misc_quicklook(this_imagename,
remove_psf=True,
remove_residual=this_niter == 0,
remove_image=True if export_fits else False)
t1 = datetime.datetime.now()
casalog.post(f"Quicklook continuum imaging took {t1 - t0}")
