def test_zero_vis_online(self):
"""Check online pipeline exits gracefully if all data is flagged
"""
# Create flagged Mock dataset and wrap it in a KatdalAdapter
ds = MockDataSet(timestamps=DEFAULT_TIMESTAMPS,
subarrays=DEFAULT_SUBARRAYS,
spws=self.spws,
dumps=self.scans,
flags=partial(flags, flagged=True))
# Dummy CB_ID and Product ID and temp fits disk
fd = kc.get_config()['fitsdirs']
fd += [(None, '/tmp/FITS')]
kc.set_config(output_id='OID', cb_id='CBID', fitsdirs=fd)
setup_aips_disks()
# Create the pipeline
pipeline = pipeline_factory('online', ds, TelescopeState(),
katdal_select=self.select,
uvblavg_params=self.uvblavg_params,
mfimage_params=self.mfimage_params)
metadata = pipeline.execute()
# Check metadata is empty and no exceptions are thrown
assert_equal(metadata, {})
# Get fits area
cfg = kc.get_config()
fits_area = cfg['fitsdirs'][-1][1]
# Remove the tmp/FITS dir
shutil.rmtree(fits_area)
def test_new_online_pipeline(self):
"""
Tests that a run of the online continuum pipeline exectues.
"""
# Create Mock dataset and wrap it in a KatdalAdapter
ds = MockDataSet(timestamps=DEFAULT_TIMESTAMPS,
subarrays=DEFAULT_SUBARRAYS,
spws=self.spws,
dumps=self.scans)
# Create a FAKE object
FAKE = object()
# Test that metadata agrees
for k, v in DEFAULT_METADATA.items():
self.assertEqual(v, getattr(ds, k, FAKE))
# Dummy CB_ID and Product ID and temp fits disk
fd = kc.get_config()['fitsdirs']
fd += [(None, '/tmp/FITS')]
kc.set_config(output_id='OID', cb_id='CBID', fitsdirs=fd)
setup_aips_disks()
# Create the pipeline
pipeline = pipeline_factory('online', ds, TelescopeState(),
katdal_select=self.select,
uvblavg_params=self.uvblavg_params,
mfimage_params=self.mfimage_params)
metadata = pipeline.execute()
# Check that output FITS files exist and have the right names
cfg = kc.get_config()
cb_id = cfg['cb_id']
out_id = cfg['output_id']
fits_area = cfg['fitsdirs'][-1][1]
for otarg in self.sanitised_target_names:
out_strings = [cb_id, out_id, otarg, IMG_CLASS]
filename = '_'.join(filter(None, out_strings)) + '.fits'
assert_in(filename, metadata['FITSImageFilename'])
filepath = os.path.join(fits_area, filename)
assert os.path.isfile(filepath)
_check_fits_headers(filepath)
# Remove the tmp/FITS dir
shutil.rmtree(fits_area)
def _source_info(self):
"""
Infer MFImage source names and file outputs.
For each source, MFImage outputs UV data and a CLEAN file.
"""
target_indices = self.ka.target_indices[:]
# Use output_id for labels
label = kc.get_config()['output_id']
# Source names
uv_sources = [s["SOURCE"][0].strip() for s in self.ka.uv_source_rows]
uv_files = [AIPSPath(name=s, disk=self.disk, label=label,
aclass=UV_CLASS, atype="UV")
for s in uv_sources]
clean_files = [AIPSPath(name=s, disk=self.disk, label=label,
aclass=IMG_CLASS, atype="MA")
for s in uv_sources]
# Find a maximum sequence number referencing unassigned
# catalogue numbers for all uv and clean files
max_uv_seq_nr = max(next_seq_nr(f) for f in uv_files)
max_clean_seq_nr = max(next_seq_nr(f) for f in clean_files)
uv_files = [f.copy(seq=max_uv_seq_nr) for f in uv_files]
clean_files = [f.copy(seq=max_clean_seq_nr) for f in clean_files]
return uv_sources, target_indices, uv_files, clean_files
def link_obit_data():
"""
Creates soft links to Obit data files within FITS directories
"""
cfg = kc.get_config()
# Directory in which Obit data file are located
obit_data_glob = pjoin(cfg['obitroot'], 'ObitSystem', 'Obit', 'share',
'data', '*')
# Data files we wish to symlink
data_files = glob.glob(obit_data_glob)
# Separate filename from full path
filenames = [os.path.split(f)[1] for f in data_files]
# In each FITS dir, create a link to each data file
for url, fitsdir in cfg['fitsdirs']:
# Fully expand link paths
link_names = [pjoin(fitsdir, f) for f in filenames]
# Remove any prior symlinks and then symlink
for data_file, link_name in zip(data_files, link_names):
if os.path.exists(link_name):
os.remove(link_name)
try:
os.symlink(data_file, link_name)
except OSError as e:
log.warn("Unable to link '{}' to '{}'\n"
"{}".format(link_name, data_file, e))
def rewrite_netsp():
"""
Rewrite `cfg.aips.aipsroot.da00/DA00/NETSP` to reference
the AIPS directories specified in our configuration
"""
cfg = kc.get_config()
netsp = pjoin(cfg['da00'], 'NETSP')
backup = pjoin(cfg['da00'], '.NETSP.BAK')
if not os.path.exists(netsp):
log.warning(
"Could not find '%s' for modification. "
"Check your AIPS directory root '%s'.", netsp, cfg['aipsroot'])
return
# Make a copy of the original
shutil.copy(netsp, backup)
# Read the copy, writing back to the original
with open(backup, "r") as rf, open(netsp, "w") as wf:
# Retain comments
for line in rf:
if line.startswith('#'):
wf.write(line)
# Write out AIPS Directory parameters
for url, aipsdir in cfg['aipsdirs']:
log.info("Adding AIPS Disk '%s to '%s'", aipsdir, netsp)
wf.write(aipsdir +
' 365.0 0 0 0 0 0 0 0 0\n')
# Remove the copy
os.remove(backup)
def __init__(self):
"""
Constructor
Largely derived from
https://github.com/bill-cotton/Obit/blob/master/ObitSystem/Obit/share/scripts/AIPSSetup.py
"""
# Get the current configuration
cfg = kc.get_config()
self.err = err = OErr.OErr()
self.obitsys = OSystem.OSystem("Pipeline", 1, cfg['userno'], 0, [" "],
0, [" "], True, False, err)
OErr.printErrMsg(err, "Error starting Obit System")
# Setup AIPS userno
AIPS.userno = cfg['userno']
# Setup Obit Environment
ObitTalkUtil.SetEnviron(AIPS_ROOT=cfg['aipsroot'],
AIPS_VERSION=cfg['aipsversion'],
OBIT_EXEC=cfg['obitexec'],
DA00=cfg['da00'],
ARCH="LINUX",
aipsdirs=cfg['aipsdirs'],
fitsdirs=cfg['fitsdirs'])
def rewrite_dadevs():
"""
Rewrite ``cfg.aips.aipsroot/DA00/DADEVS.LIST`` to reference
the AIPS directories specified in our configuration
"""
cfg = kc.get_config()
dadevs_list = pjoin(cfg['da00'], 'DADEVS.LIST')
backup = pjoin(cfg['da00'], '.DADEVS.LIST.BAK')
if not os.path.exists(dadevs_list):
log.warning(
"Could not find '%s' for modification. "
"Check your AIPS directory root '%s'.", dadevs_list,
cfg['aipsroot'])
return
# Make a copy of the original
shutil.copy(dadevs_list, backup)
# Read the copy, writing back to the original
with open(backup, "r") as rf, open(dadevs_list, "w") as wf:
# Retain comments
for line in rf:
if line.startswith('#'):
wf.write(line)
# Write out AIPS directories
for url, aipsdir in cfg['aipsdirs']:
log.info("Adding AIPS Disk '%s' to '%s'", aipsdir, dadevs_list)
wf.write("- " + aipsdir + '\n')
# Remove the copy
os.remove(backup)
def __init__(self, katdata, telstate, uvblavg_params={}, mfimage_params={},
katdal_select={}, nvispio=10240):
"""
Initialise the Continuum Pipeline for MeerKAT system processing.
Parameters
----------
katdata : :class:`katdal.Dataset`
katdal Dataset object
telstate : :class:`katsdptelstate.TelescopeState`
Telescope state or Telescope state view
uvblavg_params : dict
Dictionary of UV baseline averaging task parameters
mfimage_params : dict
Dictionary of MFImage task parameters
katdal_select : dict
Dictionary of katdal selection statements.
nvispio : integer
Number of AIPS visibilities per IO operation.
"""
super(OnlinePipeline, self).__init__(katdata)
self.telstate = telstate
self.uvblavg_params.update(uvblavg_params)
self.mfimage_params.update(mfimage_params)
self.katdal_select = katdal_select
self.nvispio = nvispio
# Use highest numbered FITS disk for FITS output.
self.odisk = len(kc.get_config()['fitsdirs'])
self.prtlv = 2
self.disk = 1
def execute_implementation(self):
result_tuple = self._select_and_infer_files()
uv_sources, target_indices, uv_files, clean_files = result_tuple
if "mfimage" in self.clobber:
self.cleanup_uv_files += uv_files
if "clean" in self.clobber:
self.cleanup_img_files += clean_files
# Update MFImage source selection
self.mfimage_params['Sources'] = uv_sources
# Find the highest numbered merge file if we are reusing
if self.reuse:
uv_mp = self.ka.aips_path(aclass='merge', name=kc.get_config()['cb_id'])
# Find the merge file with the highest seq #
hiseq = next_seq_nr(uv_mp) - 1
# hiseq will be zero if the aipsdisk has no 'merge' file
if hiseq == 0:
raise ValueError("AIPS disk at '%s' has no 'merge' file to reuse." %
(kc.get_config()['aipsdirs'][self.disk - 1][-1]))
else:
# Get the AIPS entry of the UV data to reuse
self.uv_merge_path = uv_mp.copy(seq=hiseq)
log.info("Re-using UV data in '%s' from AIPS disk: '%s'",
self.uv_merge_path, kc.get_config()['aipsdirs'][self.disk - 1][-1])
merge_uvf = uv_factory(aips_path=self.uv_merge_path, mode='r',
nvispio=self.nvispio)
merge_nvis = merge_uvf.nvis_from_NX()
else:
merge_nvis = self._export_and_merge_scans()
if "merge" in self.clobber:
self.cleanup_uv_files.append(self.uv_merge_path)
log.info('There are %s visibilities in the merged file', merge_nvis)
if merge_nvis < 1:
return {}
else:
self._run_mfimage(self.uv_merge_path, uv_sources)
self._get_wavg_img(clean_files)
for uv, clean in zip(uv_files, clean_files):
self._attach_SN_tables_to_image(uv, clean)
metadata = export_images(clean_files, target_indices, self.odisk, self.ka)
return metadata
def __init__(self, katdata, uvblavg_params={}, mfimage_params={},
katdal_select={}, nvispio=10240, prtlv=2,
clobber=set(['scans', 'avgscans']), time_step=20, reuse=False):
"""
Initialise the Continuum Pipeline for offline imaging
using a katdal dataset.
Parameters
----------
katdata : :class:`katdal.Dataset`
katdal Dataset object
uvblavg_params : dict
Dictionary of UV baseline averaging task parameters
mfimage_params : dict
Dictionary of MFImage task parameters
katdal_select : dict
Dictionary of katdal selection statements.
nvispio : integer
Number of AIPS visibilities per IO operation.
prtlv : integer
Chattiness of Obit tasks
clobber : set or iterable
Set or iterable of output files to clobber from the aipsdisk.
Possible values include:
1. `'scans'`, UV data files containing observational
data for individual scans.
2. `'avgscans'`, UV data files containing time-dependent
baseline data for individual scans.
3. `'merge'`, UV data file containing merged averaged scans.
4. `'clean'`, Output images from MFImage.
5. `'mfimage'`, Output UV data file from MFImage."
time_step : int
Size of time chunks to read vis, weights & flags from katdata.
reuse : bool
Are we reusing a previous katdal export in the aipsdisk?
"""
super(KatdalOfflinePipeline, self).__init__(katdata)
self.uvblavg_params = uvblavg_params
self.mfimage_params = mfimage_params
self.katdal_select = katdal_select
self.nvispio = nvispio
self.prtlv = prtlv
self.clobber = clobber
self.reuse = reuse
self.time_step = time_step
self.odisk = len(kc.get_config()['fitsdirs'])
def setup_aips_disks():
"""
Ensure that each AIPS and FITS disk (directory) exists.
Creates a SPACE file within the aips disks.
"""
cfg = kc.get_config()
for url, aipsdir in cfg['aipsdirs']:
# Create directory if it doesn't exist
if not os.path.exists(aipsdir):
log.info("Creating AIPS Disk '%s'", aipsdir)
os.makedirs(aipsdir)
# Create SPACE file
space = os.path.join(aipsdir, 'SPACE')
with open(space, 'a'):
os.utime(space, None)
for url, fitsdir in cfg['fitsdirs']:
# Create directory if it doesn't exist
if not os.path.exists(fitsdir):
log.info("Creating FITS Disk '%s'", fitsdir)
os.makedirs(fitsdir)
def main():
parser = create_parser()
args = parser.parse_args()
configure_logging(args)
log.info("Reading data with applycal=%s", args.applycal)
katdata = katdal.open(args.katdata,
applycal=args.applycal,
**args.open_args)
# Apply the supplied mask to the flags
if args.mask:
apply_user_mask(katdata, args.mask)
# Set up katdal selection based on arguments
kat_select = {'pol': args.pols, 'nif': args.nif}
if args.targets:
kat_select['targets'] = args.targets
if args.channels:
start_chan, end_chan = args.channels
kat_select['channels'] = slice(start_chan, end_chan)
# Command line katdal selection overrides command line options
kat_select = recursive_merge(args.select, kat_select)
# Get band and determine default .yaml files
band = katdata.spectral_windows[katdata.spw].band
uvblavg_parm_file = args.uvblavg_config
if not uvblavg_parm_file:
uvblavg_parm_file = os.path.join(os.sep, "obitconf",
f"uvblavg_{band}.yaml")
log.info('UVBlAvg parameter file for %s-band: %s', band, uvblavg_parm_file)
mfimage_parm_file = args.mfimage_config
if not mfimage_parm_file:
mfimage_parm_file = os.path.join(os.sep, "obitconf",
f"mfimage_{band}.yaml")
log.info('MFImage parameter file for %s-band: %s', band, mfimage_parm_file)
# Get defaults for uvblavg and mfimage and merge user supplied ones
uvblavg_args = get_and_merge_args(uvblavg_parm_file, args.uvblavg)
mfimage_args = get_and_merge_args(mfimage_parm_file, args.mfimage)
# Grab the cal refant from the katdal dataset and default to
# it if it is available and hasn't been set by the user.
ts = katdata.source.telstate
refant = ts.get('cal_refant')
if refant is not None and 'refAnt' not in mfimage_args:
mfimage_args['refAnt'] = aips_ant_nr(refant)
# Try and always average down to 1024 channels if the user
# hasn't specified something else
num_chans = len(katdata.channels)
factor = num_chans // 1024
if 'avgFreq' not in uvblavg_args:
if factor > 1:
uvblavg_args['avgFreq'] = 1
uvblavg_args['chAvg'] = factor
# Get the default config.
dc = kc.get_config()
# capture_block_id is used to generate AIPS disk filenames
capture_block_id = katdata.obs_params['capture_block_id']
if args.reuse:
# Set up AIPS disk from specified directory
if os.path.exists(args.reuse):
aipsdirs = [(None, args.reuse)]
log.info('Re-using AIPS data area: %s', aipsdirs[0][1])
reuse = True
else:
msg = "AIPS disk at '%s' does not exist." % (args.reuse)
log.exception(msg)
raise IOError(msg)
else:
# Set up aipsdisk configuration from args.workdir
aipsdirs = [(None,
os.path.join(args.workdir,
capture_block_id + '_aipsdisk'))]
log.info('Using AIPS data area: %s', aipsdirs[0][1])
reuse = False
# Set up output configuration from args.outputdir
fitsdirs = dc['fitsdirs']
# Append outputdir to fitsdirs
fitsdirs += [(None, args.outputdir)]
log.info('Using output data area: %s', args.outputdir)
kc.set_config(aipsdirs=aipsdirs,
fitsdirs=fitsdirs,
output_id='',
cb_id=capture_block_id)
setup_aips_disks()
pipeline = pipeline_factory('offline',
katdata,
katdal_select=kat_select,
uvblavg_params=uvblavg_args,
mfimage_params=mfimage_args,
nvispio=args.nvispio,
clobber=args.clobber,
prtlv=args.prtlv,
reuse=reuse)
# Execute it
pipeline.execute()
def main():
setup_logging()
parser = create_parser()
args = parser.parse_args()
# Open the observation
if (args.access_key is not None) != (args.secret_key is not None):
parser.error('--access-key and --secret-key must be used together')
if args.access_key is not None and args.token is not None:
parser.error('--access-key/--secret-key cannot be used with --token')
open_kwargs = {}
if args.access_key is not None:
open_kwargs['credentials'] = (args.access_key, args.secret_key)
elif args.token is not None:
open_kwargs['token'] = args.token
katdata = katdal.open(args.katdata, applycal='l1', **open_kwargs)
post_process_args(args, katdata)
uvblavg_args, mfimage_args, band = _infer_defaults_from_katdal(katdata)
# Get config defaults for uvblavg and mfimage and merge user supplied ones
uvblavg_parm_file = pjoin(CONFIG, f'uvblavg_MKAT_{band}.yaml')
log.info('UVBlAvg parameter file for %s-band: %s', band, uvblavg_parm_file)
mfimage_parm_file = pjoin(CONFIG, f'mfimage_MKAT_{band}.yaml')
log.info('MFImage parameter file for %s-band: %s', band, mfimage_parm_file)
user_uvblavg_args = get_and_merge_args(uvblavg_parm_file, args.uvblavg)
user_mfimage_args = get_and_merge_args(mfimage_parm_file, args.mfimage)
# Merge katdal defaults with user supplied defaults
recursive_merge(user_uvblavg_args, uvblavg_args)
recursive_merge(user_mfimage_args, mfimage_args)
# Get the default config.
dc = kc.get_config()
# Set up aipsdisk configuration from args.workdir
if args.workdir is not None:
aipsdirs = [(None,
pjoin(args.workdir, args.capture_block_id + '_aipsdisk'))]
else:
aipsdirs = dc['aipsdirs']
log.info('Using AIPS data area: %s', aipsdirs[0][1])
# Set up output configuration from args.outputdir
fitsdirs = dc['fitsdirs']
outputname = args.capture_block_id + OUTDIR_SEPARATOR + args.telstate_id + \
OUTDIR_SEPARATOR + START_TIME
outputdir = pjoin(args.outputdir, outputname)
# Set writing tag for duration of the pipeline
work_outputdir = outputdir + WRITE_TAG
# Append outputdir to fitsdirs
# NOTE: Pipeline is set up to always place its output in the
# highest numbered fits disk so we ensure that is the case
# here.
fitsdirs += [(None, work_outputdir)]
log.info('Using output data area: %s', outputdir)
kc.set_config(aipsdirs=aipsdirs, fitsdirs=fitsdirs)
setup_aips_disks()
# Add output_id and capture_block_id to configuration
kc.set_config(cfg=kc.get_config(),
output_id=args.output_id,
cb_id=args.capture_block_id)
# Set up telstate link then create
# a view based the capture block ID and output ID
telstate = TelescopeState(args.telstate)
view = telstate.join(args.capture_block_id, args.telstate_id)
ts_view = telstate.view(view)
katdal_select = args.select
katdal_select['nif'] = args.nif
# Create Continuum Pipeline
pipeline = pipeline_factory('online',
katdata,
ts_view,
katdal_select=katdal_select,
uvblavg_params=uvblavg_args,
mfimage_params=mfimage_args,
nvispio=args.nvispio)
# Execute it
metadata = pipeline.execute()
# Create QA products if images were created
if metadata:
make_pbeam_images(metadata, outputdir, WRITE_TAG)
make_qa_report(metadata, outputdir, WRITE_TAG)
organise_qa_output(metadata, outputdir, WRITE_TAG)
# Remove the writing tag from the output directory
os.rename(work_outputdir, outputdir)
else:
os.rmdir(work_outputdir)
def _export_and_merge_scans(self):
"""
1. Read scans from katdal
2. Export scan data to an AIPS UV file
3. Baseline average the file.
4. Merge averaged AIPS UV file into a merge UV file.
"""
# The merged UV observation file. We wait until
# we have a baseline averaged file with which to condition it
merge_uvf = None
uv_mp = self.ka.aips_path(aclass='merge', name=kc.get_config()['cb_id'])
self.uv_merge_path = uv_mp.copy(seq=next_seq_nr(uv_mp))
global_desc = self.ka.uv_descriptor()
global_table_cmds = self.ka.default_table_cmds()
# FORTRAN indexing
merge_firstVis = 1
# Scan indices
scan_indices = [int(si) for si in self.ka.scan_indices]
merge_blavg_nvis = 0
# Export each scan individually, baseline averaging and merging it
# into the final observation file.
# NOTE: Loop over scan indices here rather than using the ka.scans
# generator to avoid a conflict with the loop over ka.scans in uv_export.
for si in scan_indices:
# Select the current scan
self.ka.select(scans=si)
# Get path, with sequence based on scan index
scan_path = self.uv_merge_path.copy(aclass='raw', seq=int(si))
# Get the AIPS source for logging purposes
aips_source = self.ka.catalogue[self.ka.target_indices[0]]
aips_source_name = aips_source["SOURCE"][0].strip()
log.info("Creating '%s'", scan_path)
# Create a UV file for the scan and export to it
with uv_factory(aips_path=scan_path, mode="w",
nvispio=self.nvispio,
table_cmds=global_table_cmds,
desc=global_desc) as uvf:
uv_export(self.ka, uvf, time_step=self.time_step)
# Retrieve the single scan index.
# The time centroids and interval should be correct
# but the visibility indices need to be repurposed
scan_uvf = uv_factory(aips_path=scan_path, mode='r',
nvispio=self.nvispio)
assert len(scan_uvf.tables["AIPS NX"].rows) == 1
nx_row = scan_uvf.tables["AIPS NX"].rows[0].copy()
scan_nvis = scan_uvf.nvis_from_NX()
# If we should be merging scans
# just use the existing scan path and file
if self.merge_scans:
blavg_path = scan_path
blavg_uvf = scan_uvf
# Otherwise performing baseline averaging, deriving
# a new scan path and file
else:
# Perform baseline averaging
blavg_path = self._blavg_scan(scan_path)
blavg_uvf = uv_factory(aips_path=blavg_path,
mode='r',
nvispio=self.nvispio)
# Create the merge UV file, if necessary
merge_uvf = self._maybe_create_merge_uvf(merge_uvf, blavg_uvf,
global_table_cmds)
blavg_nvis = blavg_uvf.nvis_from_NX()
merge_blavg_nvis += blavg_nvis
# Record something about the baseline averaging process
param_str = ', '.join("%s=%s" % (k, v)
for k, v
in self.uvblavg_params.items())
blavg_history = ("Scan %d '%s' averaged "
"%s to %s visiblities. UVBlAvg(%s)" %
(si, aips_source_name, scan_nvis,
blavg_nvis, param_str))
log.info(blavg_history)
merge_uvf.append_history(blavg_history)
if blavg_nvis > 0:
log.info("Merging '%s' into '%s'", blavg_path, self.uv_merge_path)
merge_firstVis = self._copy_scan_to_merge(merge_firstVis,
merge_uvf, blavg_uvf,
nx_row)
else:
log.warn("No visibilities to merge for scan %d", si)
# Remove scan once merged
if 'scans' in self.clobber:
log.info("Zapping '%s'", scan_uvf.aips_path)
scan_uvf.Zap()
else:
scan_uvf.Close()
# If merging scans for testing purposes, our
# baseline averaged file will be the same as the
# scan file, which was handled above, so don't
# delete again. Otherwise default to
# normal clobber handling.
if not self.merge_scans:
if 'avgscans' in self.clobber:
log.info("Zapping '%s'", blavg_uvf.aips_path)
blavg_uvf.Zap()
else:
blavg_uvf.Close()
if merge_blavg_nvis == 0:
log.error("Final merged file '%s' has ZERO averaged visibilities",
self.uv_merge_path)
# Write the index table
merge_uvf.tables["AIPS NX"].write()
# Create an empty calibration table
merge_uvf.attach_CL_from_NX_table(self.ka.max_antenna_number)
# Close merge file
merge_uvf.close()
return merge_blavg_nvis
def test_gains_export(self):
"""Check l2 export to telstate"""
nchan = 128
nif = 4
dump_period = 1.0
centre_freq = 1200.e6
bandwidth = 100.e6
solPint = dump_period / 2.
solAint = dump_period
AP_telstate = 'product_GAMP_PHASE'
P_telstate = 'product_GPHASE'
spws = [{'centre_freq': centre_freq,
'num_chans': nchan,
'channel_width': bandwidth / nchan,
'sideband': 1,
'band': 'L'}]
ka_select = {'pol': 'HH,VV', 'scans': 'track',
'corrprods': 'cross', 'nif': nif}
uvblavg_params = {'maxFact': 1.0, 'avgFreq': 0,
'FOV': 100.0, 'maxInt': 1.e-6}
mfimage_params = {'Niter': 50, 'FOV': 0.1,
'xCells': 5., 'yCells': 5.,
'doGPU': False, 'Robust': -1.5,
'minFluxPSC': 0.1, 'solPInt': solPint / 60.,
'solPMode': 'P', 'minFluxASC': 0.1,
'solAInt': solAint / 60., 'maxFBW': 0.02}
# Simulate a '10Jy' source at the phase center
cat = katpoint.Catalogue()
cat.add(katpoint.Target(
"Alberich lord of the Nibelungs, radec, 20.0, -30.0, (856. 1712. 1. 0. 0.)"))
telstate = TelescopeState()
# Set up a scratch space in /tmp
fd = kc.get_config()['fitsdirs']
fd += [(None, '/tmp/FITS')]
kc.set_config(cb_id='CBID', fitsdirs=fd)
setup_aips_disks()
scan = [('track', 4, cat.targets[0])]
# Construct a simulated dataset with our
# point source at the centre of the field
ds = MockDataSet(timestamps={'start_time': 0.0, 'dump_period': dump_period},
subarrays=DEFAULT_SUBARRAYS,
spws=spws,
dumps=scan,
vis=partial(vis, sources=cat),
weights=weights,
flags=flags)
# Try one round of phase only self-cal & Amp+Phase self-cal
mfimage_params['maxPSCLoop'] = 1
mfimage_params['maxASCLoop'] = 1
# Run the pipeline
pipeline = pipeline_factory('online', ds, telstate, katdal_select=ka_select,
uvblavg_params=uvblavg_params,
mfimage_params=mfimage_params)
pipeline.execute()
ts = telstate.view('selfcal')
# Check what we have in telstate agrees with what we put in
self.assertEqual(len(ts['antlist']), len(ANTENNA_DESCRIPTIONS))
self.assertEqual(ts['bandwidth'], bandwidth)
self.assertEqual(ts['n_chans'], nif)
pol_ordering = [pol[0] for pol in sorted(CORR_ID_MAP, key=CORR_ID_MAP.get)
if pol[0] == pol[1]]
self.assertEqual(ts['pol_ordering'], pol_ordering)
if_width = bandwidth / nif
center_if = nif // 2
start_freq = centre_freq - (bandwidth / 2.)
self.assertEqual(ts['center_freq'], start_freq + if_width * (center_if + 0.5))
self.assertIn(ts.join('selfcal', P_telstate), ts.keys())
self.assertIn(ts.join('selfcal', AP_telstate), ts.keys())
def check_gains_timestamps(gains, expect_timestamps):
timestamps = []
for gain, timestamp in gains:
np.testing.assert_array_almost_equal(np.abs(gain), 1.0, decimal=3)
np.testing.assert_array_almost_equal(np.angle(gain), 0.0)
timestamps.append(timestamp)
np.testing.assert_array_almost_equal(timestamps, expect_timestamps, decimal=1)
# Check phase-only gains and timestamps
P_times = np.arange(solPint, ds.end_time.secs, 2. * solPint)
check_gains_timestamps(ts.get_range(P_telstate, st=0), P_times)
# Check Amp+Phase gains
AP_times = np.arange(solAint, ds.end_time.secs, 2. * solAint)
check_gains_timestamps(ts.get_range(AP_telstate, st=0), AP_times)
# Check with no Amp+Phase self-cal
mfimage_params['maxASCLoop'] = 0
telstate.clear()
pipeline = pipeline_factory('online', ds, telstate, katdal_select=ka_select,
uvblavg_params=uvblavg_params,
mfimage_params=mfimage_params)
pipeline.execute()
self.assertIn(telstate.join('selfcal', P_telstate), ts.keys())
self.assertNotIn(telstate.join('selfcal', AP_telstate), ts.keys())
# Check with no self-cal
mfimage_params['maxPSCLoop'] = 0
telstate.clear()
pipeline = pipeline_factory('online', ds, telstate, katdal_select=ka_select,
uvblavg_params=uvblavg_params,
mfimage_params=mfimage_params)
pipeline.execute()
self.assertNotIn(telstate.join('selfcal', P_telstate), ts.keys())
self.assertNotIn(telstate.join('selfcal', AP_telstate), ts.keys())
# Cleanup workspace
shutil.rmtree(fd[-1][1])
def test_cc_export(self):
"""Check CC models returned by MFImage
"""
nchan = 128
spws = [{'centre_freq': .856e9 + .856e9 / 2.,
'num_chans': nchan,
'channel_width': .856e9 / nchan,
'sideband': 1,
'band': 'L'}]
katdal_select = {'pol': 'HH,VV', 'scans': 'track',
'corrprods': 'cross'}
uvblavg_params = {'FOV': 0.2, 'avgFreq': 0,
'chAvg': 1, 'maxInt': 2.0}
cat = katpoint.Catalogue()
cat.add(katpoint.Target("Amfortas, radec, 0.0, -90.0, (856. 1712. 1. 0. 0.)"))
cat.add(katpoint.Target("Klingsor, radec, 0.0, 0.0, (856. 1712. 2. -0.7 0.1)"))
cat.add(katpoint.Target("Kundry, radec, 100.0, -35.0, (856. 1712. -1.0 1. -0.1)"))
ts = TelescopeState()
# Set up a scratch space in /tmp
fd = kc.get_config()['fitsdirs']
fd += [(None, '/tmp/FITS')]
kc.set_config(cb_id='CBID', fitsdirs=fd)
setup_aips_disks()
# Point sources with various flux models
for targ in cat:
scans = [('track', 5, targ)]
ds = MockDataSet(timestamps={'start_time': 1.0, 'dump_period': 4.0},
subarrays=DEFAULT_SUBARRAYS,
spws=spws,
dumps=scans,
vis=partial(vis, sources=[targ]),
weights=weights,
flags=flags)
# 100 clean components
mfimage_params = {'Niter': 100, 'maxFBW': 0.05,
'FOV': 0.1, 'xCells': 5.,
'yCells': 5., 'doGPU': False}
pipeline = pipeline_factory('online', ds, ts, katdal_select=katdal_select,
uvblavg_params=uvblavg_params,
mfimage_params=mfimage_params)
pipeline.execute()
# Get the fitted CCs from telstate
fit_cc = ts.get('target0_clean_components')
ts.delete('target0_clean_components')
all_ccs = katpoint.Catalogue(fit_cc['components'])
# Should have one merged and fitted component
self.assertEqual(len(all_ccs), 1)
cc = all_ccs.targets[0]
out_fluxmodel = cc.flux_model
in_fluxmodel = targ.flux_model
# Check the flux densities of the flux model in the fitted CC's
test_freqs = np.linspace(out_fluxmodel.min_freq_MHz, out_fluxmodel.max_freq_MHz, 5)
in_flux = in_fluxmodel.flux_density(test_freqs)
out_flux = out_fluxmodel.flux_density(test_freqs)
np.testing.assert_allclose(out_flux, in_flux, rtol=1.e-3)
# A field with some off axis sources to check positions
offax_cat = katpoint.Catalogue()
offax_cat.add(katpoint.Target("Titurel, radec, 100.1, -35.05, (856. 1712. 1.1 0. 0.)"))
offax_cat.add(katpoint.Target("Gurmenanz, radec, 99.9, -34.95, (856. 1712. 1. 0. 0.)"))
scans = [('track', 5, cat.targets[2])]
ds = MockDataSet(timestamps={'start_time': 1.0, 'dump_period': 4.0},
subarrays=DEFAULT_SUBARRAYS,
spws=spws,
dumps=scans,
vis=partial(vis, sources=offax_cat),
weights=weights,
flags=flags)
# Small number of CC's and high gain (not checking flux model)
mfimage_params['Niter'] = 4
mfimage_params['FOV'] = 0.2
mfimage_params['Gain'] = 0.5
mfimage_params['Robust'] = -5
pipeline = pipeline_factory('online', ds, ts, katdal_select=katdal_select,
uvblavg_params=uvblavg_params,
mfimage_params=mfimage_params)
pipeline.execute()
fit_cc = ts.get('target0_clean_components')
ts.delete('target0_clean_components')
all_ccs = katpoint.Catalogue(fit_cc['components'])
# We should have 2 merged clean components for two source positions
self.assertEqual(len(all_ccs), 2)
# Check the positions of the clean components
# These will be ordered by decreasing flux density of the inputs
# Position should be accurate to within a 5" pixel
delta_dec = np.deg2rad(5./3600.)
for model, cc in zip(offax_cat.targets, all_ccs.targets):
delta_ra = delta_dec/np.cos(model.radec()[1])
self.assertAlmostEqual(cc.radec()[0], model.radec()[0], delta=delta_ra)
self.assertAlmostEqual(cc.radec()[1], model.radec()[1], delta=delta_dec)
# Empty the scratch space
shutil.rmtree(fd[-1][1])
def test_offline_pipeline(self):
"""
Tests that a run of the offline continuum pipeline executes.
"""
# Create Mock dataset and wrap it in a KatdalAdapter
ds = MockDataSet(timestamps=DEFAULT_TIMESTAMPS,
subarrays=DEFAULT_SUBARRAYS,
spws=self.spws,
dumps=self.scans)
# Dummy CB_ID and Product ID and temp fits and aips disks
fd = kc.get_config()['fitsdirs']
fd += [(None, os.path.join(os.sep, 'tmp', 'FITS'))]
kc.set_config(output_id='OID', cb_id='CBID', fitsdirs=fd)
setup_aips_disks()
# Create and run the pipeline
pipeline = pipeline_factory('offline', ds,
katdal_select=self.select,
uvblavg_params=self.uvblavg_params,
mfimage_params=self.mfimage_params,
clobber=CLOBBER.difference({'merge'}))
pipeline.execute()
# Check that output FITS files exist and have the right names
# Now check for files
cfg = kc.get_config()
cb_id = cfg['cb_id']
out_id = cfg['output_id']
fits_area = cfg['fitsdirs'][-1][1]
out_strings = [cb_id, out_id, self.target_name, IMG_CLASS]
filename = '_'.join(filter(None, out_strings)) + '.fits'
filepath = os.path.join(fits_area, filename)
assert os.path.isfile(filepath)
_check_fits_headers(filepath)
# Remove the tmp/FITS dir
shutil.rmtree(fits_area)
ds = MockDataSet(timestamps=DEFAULT_TIMESTAMPS,
subarrays=DEFAULT_SUBARRAYS,
spws=self.spws,
dumps=self.scans)
setup_aips_disks()
# Create and run the pipeline (Reusing the previous data)
pipeline = pipeline_factory('offline', ds,
katdal_select=self.select,
uvblavg_params=self.uvblavg_params,
mfimage_params=self.mfimage_params,
reuse=True,
clobber=CLOBBER)
metadata = pipeline.execute()
assert_in(filename, metadata['FITSImageFilename'])
assert os.path.isfile(filepath)
_check_fits_headers(filepath)
# Remove FITS temporary area
shutil.rmtree(fits_area)
