def test_zero_vis_online(self): """Check online pipeline exits gracefully if all data is flagged """ # Create flagged Mock dataset and wrap it in a KatdalAdapter ds = MockDataSet(timestamps=DEFAULT_TIMESTAMPS, subarrays=DEFAULT_SUBARRAYS, spws=self.spws, dumps=self.scans, flags=partial(flags, flagged=True)) # Dummy CB_ID and Product ID and temp fits disk fd = kc.get_config()['fitsdirs'] fd += [(None, '/tmp/FITS')] kc.set_config(output_id='OID', cb_id='CBID', fitsdirs=fd) setup_aips_disks() # Create the pipeline pipeline = pipeline_factory('online', ds, TelescopeState(), katdal_select=self.select, uvblavg_params=self.uvblavg_params, mfimage_params=self.mfimage_params) metadata = pipeline.execute() # Check metadata is empty and no exceptions are thrown assert_equal(metadata, {}) # Get fits area cfg = kc.get_config() fits_area = cfg['fitsdirs'][-1][1] # Remove the tmp/FITS dir shutil.rmtree(fits_area)
def test_empty_dataset(self): """Test that a completely flagged dataset is exported without error""" nchan = 16 spws = [{ 'centre_freq': .856e9 + .856e9 / 2., 'num_chans': nchan, 'channel_width': .856e9 / nchan, 'sideband': 1, 'band': 'L', }] targets = [katpoint.Target("Flosshilde, radec, 0.0, -30.0")] # Set up a scan scans = [('track', 10, targets[0])] # Flag the data def mock_flags(dataset): return np.ones(dataset.shape, dtype=np.bool) # Create Mock dataset and wrap it in a KatdalAdapter ds = MockDataSet(timestamps=DEFAULT_TIMESTAMPS, subarrays=DEFAULT_SUBARRAYS, spws=spws, dumps=scans, flags=mock_flags) with obit_context(): pipeline = pipeline_factory('offline', ds) pipeline._select_and_infer_files() pipeline._export_and_merge_scans()
def test_new_online_pipeline(self): """ Tests that a run of the online continuum pipeline exectues. """ # Create Mock dataset and wrap it in a KatdalAdapter ds = MockDataSet(timestamps=DEFAULT_TIMESTAMPS, subarrays=DEFAULT_SUBARRAYS, spws=self.spws, dumps=self.scans) # Create a FAKE object FAKE = object() # Test that metadata agrees for k, v in DEFAULT_METADATA.items(): self.assertEqual(v, getattr(ds, k, FAKE)) # Dummy CB_ID and Product ID and temp fits disk fd = kc.get_config()['fitsdirs'] fd += [(None, '/tmp/FITS')] kc.set_config(output_id='OID', cb_id='CBID', fitsdirs=fd) setup_aips_disks() # Create the pipeline pipeline = pipeline_factory('online', ds, TelescopeState(), katdal_select=self.select, uvblavg_params=self.uvblavg_params, mfimage_params=self.mfimage_params) metadata = pipeline.execute() # Check that output FITS files exist and have the right names cfg = kc.get_config() cb_id = cfg['cb_id'] out_id = cfg['output_id'] fits_area = cfg['fitsdirs'][-1][1] for otarg in self.sanitised_target_names: out_strings = [cb_id, out_id, otarg, IMG_CLASS] filename = '_'.join(filter(None, out_strings)) + '.fits' assert_in(filename, metadata['FITSImageFilename']) filepath = os.path.join(fits_area, filename) assert os.path.isfile(filepath) _check_fits_headers(filepath) # Remove the tmp/FITS dir shutil.rmtree(fits_area)
def main(): setup_logging() parser = create_parser() args = parser.parse_args() # Open the observation if (args.access_key is not None) != (args.secret_key is not None): parser.error('--access-key and --secret-key must be used together') if args.access_key is not None and args.token is not None: parser.error('--access-key/--secret-key cannot be used with --token') open_kwargs = {} if args.access_key is not None: open_kwargs['credentials'] = (args.access_key, args.secret_key) elif args.token is not None: open_kwargs['token'] = args.token katdata = katdal.open(args.katdata, applycal='l1', **open_kwargs) post_process_args(args, katdata) uvblavg_args, mfimage_args, band = _infer_defaults_from_katdal(katdata) # Get config defaults for uvblavg and mfimage and merge user supplied ones uvblavg_parm_file = pjoin(CONFIG, f'uvblavg_MKAT_{band}.yaml') log.info('UVBlAvg parameter file for %s-band: %s', band, uvblavg_parm_file) mfimage_parm_file = pjoin(CONFIG, f'mfimage_MKAT_{band}.yaml') log.info('MFImage parameter file for %s-band: %s', band, mfimage_parm_file) user_uvblavg_args = get_and_merge_args(uvblavg_parm_file, args.uvblavg) user_mfimage_args = get_and_merge_args(mfimage_parm_file, args.mfimage) # Merge katdal defaults with user supplied defaults recursive_merge(user_uvblavg_args, uvblavg_args) recursive_merge(user_mfimage_args, mfimage_args) # Get the default config. dc = kc.get_config() # Set up aipsdisk configuration from args.workdir if args.workdir is not None: aipsdirs = [(None, pjoin(args.workdir, args.capture_block_id + '_aipsdisk'))] else: aipsdirs = dc['aipsdirs'] log.info('Using AIPS data area: %s', aipsdirs[0][1]) # Set up output configuration from args.outputdir fitsdirs = dc['fitsdirs'] outputname = args.capture_block_id + OUTDIR_SEPARATOR + args.telstate_id + \ OUTDIR_SEPARATOR + START_TIME outputdir = pjoin(args.outputdir, outputname) # Set writing tag for duration of the pipeline work_outputdir = outputdir + WRITE_TAG # Append outputdir to fitsdirs # NOTE: Pipeline is set up to always place its output in the # highest numbered fits disk so we ensure that is the case # here. fitsdirs += [(None, work_outputdir)] log.info('Using output data area: %s', outputdir) kc.set_config(aipsdirs=aipsdirs, fitsdirs=fitsdirs) setup_aips_disks() # Add output_id and capture_block_id to configuration kc.set_config(cfg=kc.get_config(), output_id=args.output_id, cb_id=args.capture_block_id) # Set up telstate link then create # a view based the capture block ID and output ID telstate = TelescopeState(args.telstate) view = telstate.join(args.capture_block_id, args.telstate_id) ts_view = telstate.view(view) katdal_select = args.select katdal_select['nif'] = args.nif # Create Continuum Pipeline pipeline = pipeline_factory('online', katdata, ts_view, katdal_select=katdal_select, uvblavg_params=uvblavg_args, mfimage_params=mfimage_args, nvispio=args.nvispio) # Execute it metadata = pipeline.execute() # Create QA products if images were created if metadata: make_pbeam_images(metadata, outputdir, WRITE_TAG) make_qa_report(metadata, outputdir, WRITE_TAG) organise_qa_output(metadata, outputdir, WRITE_TAG) # Remove the writing tag from the output directory os.rename(work_outputdir, outputdir) else: os.rmdir(work_outputdir)
def test_gains_export(self): """Check l2 export to telstate""" nchan = 128 nif = 4 dump_period = 1.0 centre_freq = 1200.e6 bandwidth = 100.e6 solPint = dump_period / 2. solAint = dump_period AP_telstate = 'product_GAMP_PHASE' P_telstate = 'product_GPHASE' spws = [{'centre_freq': centre_freq, 'num_chans': nchan, 'channel_width': bandwidth / nchan, 'sideband': 1, 'band': 'L'}] ka_select = {'pol': 'HH,VV', 'scans': 'track', 'corrprods': 'cross', 'nif': nif} uvblavg_params = {'maxFact': 1.0, 'avgFreq': 0, 'FOV': 100.0, 'maxInt': 1.e-6} mfimage_params = {'Niter': 50, 'FOV': 0.1, 'xCells': 5., 'yCells': 5., 'doGPU': False, 'Robust': -1.5, 'minFluxPSC': 0.1, 'solPInt': solPint / 60., 'solPMode': 'P', 'minFluxASC': 0.1, 'solAInt': solAint / 60., 'maxFBW': 0.02} # Simulate a '10Jy' source at the phase center cat = katpoint.Catalogue() cat.add(katpoint.Target( "Alberich lord of the Nibelungs, radec, 20.0, -30.0, (856. 1712. 1. 0. 0.)")) telstate = TelescopeState() # Set up a scratch space in /tmp fd = kc.get_config()['fitsdirs'] fd += [(None, '/tmp/FITS')] kc.set_config(cb_id='CBID', fitsdirs=fd) setup_aips_disks() scan = [('track', 4, cat.targets[0])] # Construct a simulated dataset with our # point source at the centre of the field ds = MockDataSet(timestamps={'start_time': 0.0, 'dump_period': dump_period}, subarrays=DEFAULT_SUBARRAYS, spws=spws, dumps=scan, vis=partial(vis, sources=cat), weights=weights, flags=flags) # Try one round of phase only self-cal & Amp+Phase self-cal mfimage_params['maxPSCLoop'] = 1 mfimage_params['maxASCLoop'] = 1 # Run the pipeline pipeline = pipeline_factory('online', ds, telstate, katdal_select=ka_select, uvblavg_params=uvblavg_params, mfimage_params=mfimage_params) pipeline.execute() ts = telstate.view('selfcal') # Check what we have in telstate agrees with what we put in self.assertEqual(len(ts['antlist']), len(ANTENNA_DESCRIPTIONS)) self.assertEqual(ts['bandwidth'], bandwidth) self.assertEqual(ts['n_chans'], nif) pol_ordering = [pol[0] for pol in sorted(CORR_ID_MAP, key=CORR_ID_MAP.get) if pol[0] == pol[1]] self.assertEqual(ts['pol_ordering'], pol_ordering) if_width = bandwidth / nif center_if = nif // 2 start_freq = centre_freq - (bandwidth / 2.) self.assertEqual(ts['center_freq'], start_freq + if_width * (center_if + 0.5)) self.assertIn(ts.join('selfcal', P_telstate), ts.keys()) self.assertIn(ts.join('selfcal', AP_telstate), ts.keys()) def check_gains_timestamps(gains, expect_timestamps): timestamps = [] for gain, timestamp in gains: np.testing.assert_array_almost_equal(np.abs(gain), 1.0, decimal=3) np.testing.assert_array_almost_equal(np.angle(gain), 0.0) timestamps.append(timestamp) np.testing.assert_array_almost_equal(timestamps, expect_timestamps, decimal=1) # Check phase-only gains and timestamps P_times = np.arange(solPint, ds.end_time.secs, 2. * solPint) check_gains_timestamps(ts.get_range(P_telstate, st=0), P_times) # Check Amp+Phase gains AP_times = np.arange(solAint, ds.end_time.secs, 2. * solAint) check_gains_timestamps(ts.get_range(AP_telstate, st=0), AP_times) # Check with no Amp+Phase self-cal mfimage_params['maxASCLoop'] = 0 telstate.clear() pipeline = pipeline_factory('online', ds, telstate, katdal_select=ka_select, uvblavg_params=uvblavg_params, mfimage_params=mfimage_params) pipeline.execute() self.assertIn(telstate.join('selfcal', P_telstate), ts.keys()) self.assertNotIn(telstate.join('selfcal', AP_telstate), ts.keys()) # Check with no self-cal mfimage_params['maxPSCLoop'] = 0 telstate.clear() pipeline = pipeline_factory('online', ds, telstate, katdal_select=ka_select, uvblavg_params=uvblavg_params, mfimage_params=mfimage_params) pipeline.execute() self.assertNotIn(telstate.join('selfcal', P_telstate), ts.keys()) self.assertNotIn(telstate.join('selfcal', AP_telstate), ts.keys()) # Cleanup workspace shutil.rmtree(fd[-1][1])
def test_cc_export(self): """Check CC models returned by MFImage """ nchan = 128 spws = [{'centre_freq': .856e9 + .856e9 / 2., 'num_chans': nchan, 'channel_width': .856e9 / nchan, 'sideband': 1, 'band': 'L'}] katdal_select = {'pol': 'HH,VV', 'scans': 'track', 'corrprods': 'cross'} uvblavg_params = {'FOV': 0.2, 'avgFreq': 0, 'chAvg': 1, 'maxInt': 2.0} cat = katpoint.Catalogue() cat.add(katpoint.Target("Amfortas, radec, 0.0, -90.0, (856. 1712. 1. 0. 0.)")) cat.add(katpoint.Target("Klingsor, radec, 0.0, 0.0, (856. 1712. 2. -0.7 0.1)")) cat.add(katpoint.Target("Kundry, radec, 100.0, -35.0, (856. 1712. -1.0 1. -0.1)")) ts = TelescopeState() # Set up a scratch space in /tmp fd = kc.get_config()['fitsdirs'] fd += [(None, '/tmp/FITS')] kc.set_config(cb_id='CBID', fitsdirs=fd) setup_aips_disks() # Point sources with various flux models for targ in cat: scans = [('track', 5, targ)] ds = MockDataSet(timestamps={'start_time': 1.0, 'dump_period': 4.0}, subarrays=DEFAULT_SUBARRAYS, spws=spws, dumps=scans, vis=partial(vis, sources=[targ]), weights=weights, flags=flags) # 100 clean components mfimage_params = {'Niter': 100, 'maxFBW': 0.05, 'FOV': 0.1, 'xCells': 5., 'yCells': 5., 'doGPU': False} pipeline = pipeline_factory('online', ds, ts, katdal_select=katdal_select, uvblavg_params=uvblavg_params, mfimage_params=mfimage_params) pipeline.execute() # Get the fitted CCs from telstate fit_cc = ts.get('target0_clean_components') ts.delete('target0_clean_components') all_ccs = katpoint.Catalogue(fit_cc['components']) # Should have one merged and fitted component self.assertEqual(len(all_ccs), 1) cc = all_ccs.targets[0] out_fluxmodel = cc.flux_model in_fluxmodel = targ.flux_model # Check the flux densities of the flux model in the fitted CC's test_freqs = np.linspace(out_fluxmodel.min_freq_MHz, out_fluxmodel.max_freq_MHz, 5) in_flux = in_fluxmodel.flux_density(test_freqs) out_flux = out_fluxmodel.flux_density(test_freqs) np.testing.assert_allclose(out_flux, in_flux, rtol=1.e-3) # A field with some off axis sources to check positions offax_cat = katpoint.Catalogue() offax_cat.add(katpoint.Target("Titurel, radec, 100.1, -35.05, (856. 1712. 1.1 0. 0.)")) offax_cat.add(katpoint.Target("Gurmenanz, radec, 99.9, -34.95, (856. 1712. 1. 0. 0.)")) scans = [('track', 5, cat.targets[2])] ds = MockDataSet(timestamps={'start_time': 1.0, 'dump_period': 4.0}, subarrays=DEFAULT_SUBARRAYS, spws=spws, dumps=scans, vis=partial(vis, sources=offax_cat), weights=weights, flags=flags) # Small number of CC's and high gain (not checking flux model) mfimage_params['Niter'] = 4 mfimage_params['FOV'] = 0.2 mfimage_params['Gain'] = 0.5 mfimage_params['Robust'] = -5 pipeline = pipeline_factory('online', ds, ts, katdal_select=katdal_select, uvblavg_params=uvblavg_params, mfimage_params=mfimage_params) pipeline.execute() fit_cc = ts.get('target0_clean_components') ts.delete('target0_clean_components') all_ccs = katpoint.Catalogue(fit_cc['components']) # We should have 2 merged clean components for two source positions self.assertEqual(len(all_ccs), 2) # Check the positions of the clean components # These will be ordered by decreasing flux density of the inputs # Position should be accurate to within a 5" pixel delta_dec = np.deg2rad(5./3600.) for model, cc in zip(offax_cat.targets, all_ccs.targets): delta_ra = delta_dec/np.cos(model.radec()[1]) self.assertAlmostEqual(cc.radec()[0], model.radec()[0], delta=delta_ra) self.assertAlmostEqual(cc.radec()[1], model.radec()[1], delta=delta_dec) # Empty the scratch space shutil.rmtree(fd[-1][1])
def test_offline_pipeline(self): """ Tests that a run of the offline continuum pipeline executes. """ # Create Mock dataset and wrap it in a KatdalAdapter ds = MockDataSet(timestamps=DEFAULT_TIMESTAMPS, subarrays=DEFAULT_SUBARRAYS, spws=self.spws, dumps=self.scans) # Dummy CB_ID and Product ID and temp fits and aips disks fd = kc.get_config()['fitsdirs'] fd += [(None, os.path.join(os.sep, 'tmp', 'FITS'))] kc.set_config(output_id='OID', cb_id='CBID', fitsdirs=fd) setup_aips_disks() # Create and run the pipeline pipeline = pipeline_factory('offline', ds, katdal_select=self.select, uvblavg_params=self.uvblavg_params, mfimage_params=self.mfimage_params, clobber=CLOBBER.difference({'merge'})) pipeline.execute() # Check that output FITS files exist and have the right names # Now check for files cfg = kc.get_config() cb_id = cfg['cb_id'] out_id = cfg['output_id'] fits_area = cfg['fitsdirs'][-1][1] out_strings = [cb_id, out_id, self.target_name, IMG_CLASS] filename = '_'.join(filter(None, out_strings)) + '.fits' filepath = os.path.join(fits_area, filename) assert os.path.isfile(filepath) _check_fits_headers(filepath) # Remove the tmp/FITS dir shutil.rmtree(fits_area) ds = MockDataSet(timestamps=DEFAULT_TIMESTAMPS, subarrays=DEFAULT_SUBARRAYS, spws=self.spws, dumps=self.scans) setup_aips_disks() # Create and run the pipeline (Reusing the previous data) pipeline = pipeline_factory('offline', ds, katdal_select=self.select, uvblavg_params=self.uvblavg_params, mfimage_params=self.mfimage_params, reuse=True, clobber=CLOBBER) metadata = pipeline.execute() assert_in(filename, metadata['FITSImageFilename']) assert os.path.isfile(filepath) _check_fits_headers(filepath) # Remove FITS temporary area shutil.rmtree(fits_area)
def _test_export_implementation(self, export_type="uv_export", nif=1): """ Implementation of export test. Tests export via either the :func:`katacomb.uv_export` or :func:`katacomb.pipeline_factory, depending on ``export_type``. When testing export via the Continuum Pipeline, baseline averaging is disabled. Parameters ---------- export_type (optional): string Either ``"uv_export"`` or ``"continuum_export"``. Defaults to ``"uv_export"`` nif (optional): nif Number of IFs to test splitting the band into """ nchan = 16 nvispio = 1024 spws = [{ 'centre_freq': .856e9 + .856e9 / 2., 'num_chans': nchan, 'channel_width': .856e9 / nchan, 'sideband': 1, 'band': 'L', }] target_names = random.sample(stars.keys(), 5) # Pick 5 random stars as targets targets = [katpoint.Target("%s, star" % t) for t in target_names] # Set up varying scans scans = [('slew', 1, targets[0]), ('track', 3, targets[0]), ('slew', 2, targets[1]), ('track', 5, targets[1]), ('slew', 1, targets[2]), ('track', 8, targets[2]), ('slew', 2, targets[3]), ('track', 9, targets[3]), ('slew', 1, targets[4]), ('track', 10, targets[4])] # Create Mock dataset and wrap it in a KatdalAdapter ds = MockDataSet(timestamps=DEFAULT_TIMESTAMPS, subarrays=DEFAULT_SUBARRAYS, spws=spws, dumps=scans) KA = KatdalAdapter(ds) # Create a FAKE object FAKE = object() # Test that metadata agrees for k, v in DEFAULT_METADATA.items(): self.assertEqual(v, getattr(KA, k, FAKE)) # Setup the katdal selection, convert it to a string # accepted by our command line parser function, which # converts it back to a dict. select = { 'scans': 'track', 'corrprods': 'cross', 'targets': target_names, 'pol': 'HH,VV', 'channels': slice(0, nchan), } assign_str = '; '.join('%s=%s' % (k, repr(v)) for k, v in select.items()) select = parse_python_assigns(assign_str) # Add nif to selection if nif > 1: select['nif'] = nif # Perform the katdal selection KA.select(**select) # Obtain correlator products and produce argsorts that will # order by (a1, a2, stokes) cp = KA.correlator_products() nstokes = KA.nstokes # Lexicographically sort correlation products on (a1, a2, cid) def sort_fn(x): return (cp[x].ant1_ix, cp[x].ant2_ix, cp[x].cid) cp_argsort = np.asarray(sorted(range(len(cp)), key=sort_fn)) # Use first stokes parameter index of each baseline bl_argsort = cp_argsort[::nstokes] # Get data shape after selection kat_ndumps, kat_nchans, kat_ncorrprods = KA.shape uv_file_path = AIPSPath('test', 1, 'test', 1) with obit_context(), file_cleaner([uv_file_path]): # Perform export of katdal selection via uv_export if export_type == "uv_export": with uv_factory(aips_path=uv_file_path, mode="w", nvispio=nvispio, table_cmds=KA.default_table_cmds(), desc=KA.uv_descriptor()) as uvf: uv_export(KA, uvf) # Perform export of katdal selection via ContinuumPipline elif export_type == "continuum_export": pipeline = pipeline_factory(export_type, KA.katdal, katdal_select=select, merge_scans=True) pipeline._select_and_infer_files() pipeline._export_and_merge_scans() uv_file_path = pipeline.uv_merge_path newselect = select.copy() newselect['reset'] = 'TFB' KA.select(**newselect) else: raise ValueError("Invalid export_type '%s'" % export_type) nvispio = 1 # Now read from the AIPS UV file and sanity check with uv_factory(aips_path=uv_file_path, mode="r", nvispio=nvispio) as uvf: def _strip_strings(aips_keywords): """ AIPS string are padded, strip them """ return {k: v.strip() if isinstance(v, (str, bytes)) else v for k, v in aips_keywords.items()} fq_kw = _strip_strings(uvf.tables["AIPS FQ"].keywords) src_kw = _strip_strings(uvf.tables["AIPS SU"].keywords) ant_kw = _strip_strings(uvf.tables["AIPS AN"].keywords) # Check that the subset of keywords generated # by the katdal adapter match those read from the AIPS table self.assertDictContainsSubset(KA.uv_spw_keywords, fq_kw) self.assertDictContainsSubset(KA.uv_source_keywords, src_kw) self.assertDictContainsSubset(KA.uv_antenna_keywords, ant_kw) def _strip_metadata(aips_table_rows): """ Strip out ``Numfields``, ``_status``, ``Table name`` fields from each row entry """ STRIP = {'NumFields', '_status', 'Table name'} return [{k: v for k, v in d.items() if k not in STRIP} for d in aips_table_rows] # Check that frequency, source and antenna rows # are correctly exported fq_rows = _strip_metadata(uvf.tables["AIPS FQ"].rows) self.assertEqual(fq_rows, KA.uv_spw_rows) ant_rows = _strip_metadata(uvf.tables["AIPS AN"].rows) self.assertEqual(ant_rows, KA.uv_antenna_rows) # TODO(sjperkins) # For some reason, source radec and apparent radec # coordinates are off by some minor difference # Probably related to float32 conversion. if not export_type == "continuum_export": src_rows = _strip_metadata(uvf.tables["AIPS SU"].rows) self.assertEqual(src_rows, KA.uv_source_rows) uv_desc = uvf.Desc.Dict inaxes = tuple(reversed(uv_desc['inaxes'][:6])) naips_vis = uv_desc['nvis'] summed_vis = 0 # Number of random parameters nrparm = uv_desc['nrparm'] # Length of visibility buffer record lrec = uv_desc['lrec'] # Random parameter indices ilocu = uv_desc['ilocu'] # U ilocv = uv_desc['ilocv'] # V ilocw = uv_desc['ilocw'] # W iloct = uv_desc['iloct'] # time ilocsu = uv_desc['ilocsu'] # source id # Sanity check the UV descriptor inaxes uv_nra, uv_ndec, uv_nif, uv_nchans, uv_nstokes, uv_viscomp = inaxes self.assertEqual(uv_nchans * uv_nif, kat_nchans, "Number of AIPS and katdal channels differ") self.assertEqual(uv_viscomp, 3, "Number of AIPS visibility components") self.assertEqual(uv_nra, 1, "RA should be 1") self.assertEqual(uv_ndec, 1, "DEC should be 1") self.assertEqual(uv_nif, nif, "NIF should be %d" % (nif)) # Compare AIPS and katdal scans aips_scans = uvf.tables["AIPS NX"].rows katdal_scans = list(KA.scans()) # Must have same number of scans self.assertEqual(len(aips_scans), len(katdal_scans)) # Iterate through the katdal scans for i, (si, state, target) in enumerate(KA.scans()): self.assertTrue(state in select['scans']) kat_ndumps, kat_nchans, kat_ncorrprods = KA.shape # Was is the expected source ID? expected_source = np.float32(target['ID. NO.'][0]) # Work out start, end and length of the scan # in visibilities aips_scan = aips_scans[i] start_vis = aips_scan['START VIS'][0] last_vis = aips_scan['END VIS'][0] naips_scan_vis = last_vis - start_vis + 1 summed_vis += naips_scan_vis # Each AIPS visibility has dimension [1,1,1,nchan,nstokes,3] # and one exists for each timestep and baseline # Ensure that the number of visibilities equals # number of dumps times number of baselines self.assertEqual(naips_scan_vis, kat_ndumps*kat_ncorrprods//uv_nstokes, 'Mismatch in number of visibilities in scan %d' % si) # Accumulate UVW, time data from the AIPS UV file # By convention uv_export's data in (ntime, nbl) # ordering, so we assume that the AIPS UV data # is ordered the same way u_data = [] v_data = [] w_data = [] time_data = [] vis_data = [] # For each visibility in the scan, read data and # compare with katdal observation data for firstVis in range(start_vis, last_vis+1, nvispio): # Determine number of visibilities to read numVisBuff = min(last_vis+1-firstVis, nvispio) desc = uvf.Desc.Dict desc.update(numVisBuff=numVisBuff) uvf.Desc.Dict = desc # Read a visibility uvf.Read(firstVis=firstVis) buf = uvf.np_visbuf # Must copy because buf data will change with each read u_data.append(buf[ilocu:lrec*numVisBuff:lrec].copy()) v_data.append(buf[ilocv:lrec*numVisBuff:lrec].copy()) w_data.append(buf[ilocw:lrec*numVisBuff:lrec].copy()) time_data.append(buf[iloct:lrec*numVisBuff:lrec].copy()) for i in range(numVisBuff): base = nrparm + i*lrec data = buf[base:base+lrec-nrparm].copy() data = data.reshape(inaxes) vis_data.append(data) # Check that we're dealing with the same source # within the scan sources = buf[ilocsu:lrec*numVisBuff:lrec].copy() self.assertEqual(sources, expected_source) # Ensure katdal timestamps match AIPS UV file timestamps # and that there are exactly number of baseline counts # for each one times, time_counts = np.unique(time_data, return_counts=True) timestamps = KA.uv_timestamps[:].astype(np.float32) self.assertTrue(np.all(times == timestamps)) self.assertTrue(np.all(time_counts == len(bl_argsort))) # Flatten AIPS UVW data, there'll be (ntime*nbl) values u_data = np.concatenate(u_data).ravel() v_data = np.concatenate(v_data).ravel() w_data = np.concatenate(w_data).ravel() # uv_u will have shape (ntime, ncorrprods) # Select katdal stokes 0 UVW coordinates and flatten uv_u = KA.uv_u[:, bl_argsort].astype(np.float32).ravel() uv_v = KA.uv_v[:, bl_argsort].astype(np.float32).ravel() uv_w = KA.uv_w[:, bl_argsort].astype(np.float32).ravel() # Confirm UVW coordinate equality self.assertTrue(np.all(uv_u == u_data)) self.assertTrue(np.all(uv_v == v_data)) self.assertTrue(np.all(uv_w == w_data)) # Number of baselines nbl = len(bl_argsort) # Now compare visibility data # Stacking produces # (ntime*nbl, nra, ndec, nif, nchan, nstokes, 3) aips_vis = np.stack(vis_data, axis=0) kat_vis = KA.uv_vis[:] shape = (kat_ndumps, kat_nchans, nbl, nstokes, 3) # This produces (ntime, nchan, nbl, nstokes, 3) kat_vis = kat_vis[:, :, cp_argsort, :].reshape(shape) # (1) transpose so that we have (ntime, nbl, nchan, nstokes, 3) # (2) reshape to include the full inaxes shape, # including singleton nif, ra and dec dimensions kat_vis = (kat_vis.transpose(0, 2, 1, 3, 4) .reshape((kat_ndumps, nbl,) + inaxes)) aips_vis = aips_vis.reshape((kat_ndumps, nbl) + inaxes) self.assertTrue(np.all(aips_vis == kat_vis)) # Check that we read the expected number of visibilities self.assertEqual(summed_vis, naips_vis)
def main(): parser = create_parser() args = parser.parse_args() configure_logging(args) log.info("Reading data with applycal=%s", args.applycal) katdata = katdal.open(args.katdata, applycal=args.applycal, **args.open_args) # Apply the supplied mask to the flags if args.mask: apply_user_mask(katdata, args.mask) # Set up katdal selection based on arguments kat_select = {'pol': args.pols, 'nif': args.nif} if args.targets: kat_select['targets'] = args.targets if args.channels: start_chan, end_chan = args.channels kat_select['channels'] = slice(start_chan, end_chan) # Command line katdal selection overrides command line options kat_select = recursive_merge(args.select, kat_select) # Get band and determine default .yaml files band = katdata.spectral_windows[katdata.spw].band uvblavg_parm_file = args.uvblavg_config if not uvblavg_parm_file: uvblavg_parm_file = os.path.join(os.sep, "obitconf", f"uvblavg_{band}.yaml") log.info('UVBlAvg parameter file for %s-band: %s', band, uvblavg_parm_file) mfimage_parm_file = args.mfimage_config if not mfimage_parm_file: mfimage_parm_file = os.path.join(os.sep, "obitconf", f"mfimage_{band}.yaml") log.info('MFImage parameter file for %s-band: %s', band, mfimage_parm_file) # Get defaults for uvblavg and mfimage and merge user supplied ones uvblavg_args = get_and_merge_args(uvblavg_parm_file, args.uvblavg) mfimage_args = get_and_merge_args(mfimage_parm_file, args.mfimage) # Grab the cal refant from the katdal dataset and default to # it if it is available and hasn't been set by the user. ts = katdata.source.telstate refant = ts.get('cal_refant') if refant is not None and 'refAnt' not in mfimage_args: mfimage_args['refAnt'] = aips_ant_nr(refant) # Try and always average down to 1024 channels if the user # hasn't specified something else num_chans = len(katdata.channels) factor = num_chans // 1024 if 'avgFreq' not in uvblavg_args: if factor > 1: uvblavg_args['avgFreq'] = 1 uvblavg_args['chAvg'] = factor # Get the default config. dc = kc.get_config() # capture_block_id is used to generate AIPS disk filenames capture_block_id = katdata.obs_params['capture_block_id'] if args.reuse: # Set up AIPS disk from specified directory if os.path.exists(args.reuse): aipsdirs = [(None, args.reuse)] log.info('Re-using AIPS data area: %s', aipsdirs[0][1]) reuse = True else: msg = "AIPS disk at '%s' does not exist." % (args.reuse) log.exception(msg) raise IOError(msg) else: # Set up aipsdisk configuration from args.workdir aipsdirs = [(None, os.path.join(args.workdir, capture_block_id + '_aipsdisk'))] log.info('Using AIPS data area: %s', aipsdirs[0][1]) reuse = False # Set up output configuration from args.outputdir fitsdirs = dc['fitsdirs'] # Append outputdir to fitsdirs fitsdirs += [(None, args.outputdir)] log.info('Using output data area: %s', args.outputdir) kc.set_config(aipsdirs=aipsdirs, fitsdirs=fitsdirs, output_id='', cb_id=capture_block_id) setup_aips_disks() pipeline = pipeline_factory('offline', katdata, katdal_select=kat_select, uvblavg_params=uvblavg_args, mfimage_params=mfimage_args, nvispio=args.nvispio, clobber=args.clobber, prtlv=args.prtlv, reuse=reuse) # Execute it pipeline.execute()