def test_readwriteskymodel_no_image(self): vis = create_blockvisibility( self.midcore, self.times, self.frequency, channel_bandwidth=self.channel_bandwidth, phasecentre=self.phasecentre, polarisation_frame=PolarisationFrame("linear"), weight=1.0) gt = create_gaintable_from_blockvisibility(vis, timeslice='auto') gt = simulate_gaintable(gt, phase_error=1.0, amplitude_error=0.1) sm = SkyModel(components=[self.comp], gaintable=gt) config = { "buffer": { "directory": self.dir }, "skymodel": { "name": "test_bufferskymodel.hdf", "data_model": "SkyModel" } } bdm = BufferSkyModel(config["buffer"], config["skymodel"], sm) bdm.sync() new_bdm = BufferSkyModel(config["buffer"], config["skymodel"]) new_bdm.sync() newsm = bdm.memory_data_model assert newsm.components[0].flux.shape == self.comp.flux.shape assert newsm.gaintable.data.shape == gt.data.shape
def test_readwritegaintable(self): self.vis = create_blockvisibility( self.midcore, self.times, self.frequency, channel_bandwidth=self.channel_bandwidth, phasecentre=self.phasecentre, polarisation_frame=PolarisationFrame("linear"), weight=1.0) gt = create_gaintable_from_blockvisibility(self.vis, timeslice='auto') gt = simulate_gaintable(gt, phase_error=1.0, amplitude_error=0.1) config = { "buffer": { "directory": self.dir }, "gaintable": { "name": "test_buffergaintable.hdf", "data_model": "GainTable" } } bdm = BufferGainTable(config["buffer"], config["gaintable"], gt) bdm.sync() new_bdm = BufferGainTable(config["buffer"], config["gaintable"]) new_bdm.sync() newgt = bdm.memory_data_model assert gt.data.shape == newgt.data.shape assert numpy.max(numpy.abs(gt.gain - newgt.gain)) < 1e-15
def test_plot_gaintable_scalar(self): self.actualSetup('stokesI', 'stokesI', f=[100.0]) gt = create_gaintable_from_blockvisibility(self.vis) log.info("Created gain table: %s" % (gaintable_summary(gt))) gt = simulate_gaintable(gt, phase_error=0.1, amplitude_error=0.1) plt.clf() gaintable_plot(gt) plt.show(block=False)
def test_apply_gaintable_only(self): for spf, dpf in[('stokesI', 'stokesI'), ('stokesIQUV', 'linear'), ('stokesIQUV', 'circular')]: self.actualSetup(spf, dpf) gt = create_gaintable_from_blockvisibility(self.vis, timeslice='auto') log.info("Created gain table: %s" % (gaintable_summary(gt))) gt = simulate_gaintable(gt, phase_error=0.1, amplitude_error=0.01) original = copy_visibility(self.vis) vis = apply_gaintable(self.vis, gt) error = numpy.max(numpy.abs(vis.vis - original.vis)) assert error > 10.0, "Error = %f" % (error)
def test_solve_gaintable_scalar_timeslice(self): self.actualSetup('stokesI', 'stokesI', f=[100.0], ntimes=10) gt = create_gaintable_from_blockvisibility(self.vis, timeslice=120.0) log.info("Created gain table: %s" % (gaintable_summary(gt))) gt = simulate_gaintable(gt, phase_error=10.0, amplitude_error=0.0) original = copy_visibility(self.vis) self.vis = apply_gaintable(self.vis, gt) gtsol = solve_gaintable(self.vis, original, phase_only=True, niter=200) residual = numpy.max(gtsol.residual) assert residual < 3e-8, "Max residual = %s" % (residual) assert numpy.max(numpy.abs(gtsol.gain - 1.0)) > 0.1
def test_solve_gaintable_stokesI_pointsource(self): self.actualSetup('stokesI', 'stokesI', f=[100.0]) gt = create_gaintable_from_blockvisibility(self.vis) log.info("Created gain table: %s" % (gaintable_summary(gt))) gt = simulate_gaintable(gt, phase_error=10.0, amplitude_error=0.0) original = copy_visibility(self.vis) self.vis = apply_gaintable(self.vis, gt) point_vis = divide_visibility(self.vis, original) gtsol = solve_gaintable(point_vis, phase_only=False, niter=200) residual = numpy.max(gtsol.residual) assert residual < 3e-8, "Max residual = %s" % (residual) assert numpy.max(numpy.abs(gtsol.gain - 1.0)) > 0.1
def test_create_gaintable_from_visibility_interval(self): for timeslice in [10.0, 'auto', 1e5]: for spf, dpf in[('stokesIQUV', 'linear')]: self.actualSetup(spf, dpf) gt = create_gaintable_from_blockvisibility(self.vis, timeslice=timeslice) log.info("Created gain table: %s" % (gaintable_summary(gt))) gt = simulate_gaintable(gt, phase_error=1.0) original = copy_visibility(self.vis) vis = apply_gaintable(self.vis, gt) assert numpy.max(numpy.abs(original.vis)) > 0.0 assert numpy.max(numpy.abs(vis.vis)) > 0.0 assert numpy.max(numpy.abs(vis.vis - original.vis)) > 0.0
def test_solve_gaintable_stokesI_small_n_large_t(self): # Select only 6 stations self.actualSetup('stokesI', 'stokesI', f=[100.0], ntimes=4000, rmax=83) gt = create_gaintable_from_blockvisibility(self.vis) log.info("Created gain table: %s" % (gaintable_summary(gt))) gt = simulate_gaintable(gt, phase_error=10.0, amplitude_error=0.0) gt.data['gain'] = gt.gain[1, ...] original = copy_visibility(self.vis) self.vis = apply_gaintable(self.vis, gt) gtsol = solve_gaintable(self.vis, original, phase_only=True, niter=200) self.vis = apply_gaintable(self.vis, gtsol) residual = numpy.max(gtsol.residual) assert residual < 3e-8, "Max residual = %s" % (residual) assert numpy.max(numpy.abs(gtsol.gain - 1.0)) > 0.1
def corrupt_vis(vis, gt, **kwargs): if isinstance(vis, Visibility): bv = convert_visibility_to_blockvisibility(vis) else: bv = vis if gt is None: gt = create_gaintable_from_blockvisibility(bv, **kwargs) gt = simulate_gaintable(gt, **kwargs) bv = apply_gaintable(bv, gt) if isinstance(vis, Visibility): return convert_blockvisibility_to_visibility(bv) else: return bv
def test_readwritegaintable(self): self.vis = create_blockvisibility(self.mid, self.times, self.frequency, channel_bandwidth=self.channel_bandwidth, phasecentre=self.phasecentre, polarisation_frame=PolarisationFrame("linear"), weight=1.0) gt = create_gaintable_from_blockvisibility(self.vis, timeslice='auto') gt = simulate_gaintable(gt, phase_error=1.0, amplitude_error=0.1) export_gaintable_to_hdf5(gt, '%s/test_data_model_helpers_gaintable.hdf' % self.dir) newgt = import_gaintable_from_hdf5('%s/test_data_model_helpers_gaintable.hdf' % self.dir) for key in gt.data.dtype.fields: assert numpy.max(numpy.abs(newgt.data[key] - gt.data[key])) < 1e-15 assert gt.data.shape == newgt.data.shape assert numpy.max(numpy.abs(gt.gain - newgt.gain)) < 1e-15
def test_solve_gaintable_scalar_normalise(self): self.actualSetup('stokesI', 'stokesI', f=[100.0]) gt = create_gaintable_from_blockvisibility(self.vis) log.info("Created gain table: %s" % (gaintable_summary(gt))) gt = simulate_gaintable(gt, phase_error=0.0, amplitude_error=0.1) gt.data['gain'] *= 2.0 original = copy_visibility(self.vis) self.vis = apply_gaintable(self.vis, gt) gtsol = solve_gaintable(self.vis, original, phase_only=False, niter=200, normalise_gains=True) residual = numpy.max(gtsol.residual) assert residual < 3e-8, "Max residual = %s" % (residual) assert numpy.max(numpy.abs(gtsol.gain - 1.0)) > 0.1
def test_calibrate_G_function(self): self.actualSetup('stokesIQUV', 'linear', f=[100.0, 0.0, 0.0, 50.0]) # Prepare the corrupted visibility data_models gt = create_gaintable_from_blockvisibility(self.vis) log.info("Created gain table: %s" % (gaintable_summary(gt))) gt = simulate_gaintable(gt, phase_error=0.0, amplitude_error=0.1) original = copy_visibility(self.vis) self.vis = apply_gaintable(self.vis, gt) # Now get the control dictionary and calibrate controls = create_calibration_controls() controls['G']['first_selfcal'] = 0 calibrated_vis, gaintables = calibrate_chain(self.vis, original, calibration_context='G', controls=controls) residual = numpy.max(gaintables['G'].residual) assert residual < 1e-8, "Max T residual = %s" % residual
def test_solve_gaintable_stokesI_bandpass(self): self.actualSetup('stokesI', 'stokesI', f=[100.0], vnchan=128) gt = create_gaintable_from_blockvisibility(self.vis) log.info("Created gain table: %s" % (gaintable_summary(gt))) gt = simulate_gaintable(gt, phase_error=10.0, amplitude_error=0.01, smooth_channels=8) original = copy_visibility(self.vis) self.vis = apply_gaintable(self.vis, gt) gtsol = solve_gaintable(self.vis, original, phase_only=False, niter=200, damping=0.5) residual = numpy.max(gtsol.residual) assert residual < 3e-8, "Max residual = %s" % (residual) assert numpy.max(numpy.abs(gtsol.gain - 1.0)) > 0.1
def core_solve(self, spf, dpf, phase_error=0.1, amplitude_error=0.0, leakage=0.0, phase_only=True, niter=200, crosspol=False, residual_tol=1e-6, f=None, vnchan=3, timeslice='auto'): if f is None: f = [100.0, 50.0, -10.0, 40.0] self.actualSetup(spf, dpf, f=f, vnchan=vnchan) gt = create_gaintable_from_blockvisibility(self.vis, timeslice=timeslice) log.info("Created gain table: %s" % (gaintable_summary(gt))) gt = simulate_gaintable(gt, phase_error=phase_error, amplitude_error=amplitude_error, leakage=leakage) original = copy_visibility(self.vis) vis = apply_gaintable(self.vis, gt) gtsol = solve_gaintable(self.vis, original, phase_only=phase_only, niter=niter, crosspol=crosspol, tol=1e-6) vis = apply_gaintable(vis, gtsol, inverse=True) residual = numpy.max(gtsol.residual) assert residual < residual_tol, "%s %s Max residual = %s" % (spf, dpf, residual) log.debug(qa_gaintable(gt)) assert numpy.max(numpy.abs(gtsol.gain - 1.0)) > 0.1
def actualSetUp(self, nfreqwin=3, dospectral=True, dopol=False, amp_errors=None, phase_errors=None, zerow=True): if amp_errors is None: amp_errors = {'T': 0.0, 'G': 0.1} if phase_errors is None: phase_errors = {'T': 1.0, 'G': 0.0} self.npixel = 512 self.low = create_named_configuration('LOWBD2', rmax=750.0) self.freqwin = nfreqwin self.vis_list = list() self.ntimes = 1 self.times = numpy.linspace(-3.0, +3.0, self.ntimes) * numpy.pi / 12.0 self.frequency = numpy.linspace(0.8e8, 1.2e8, self.freqwin) if self.freqwin > 1: self.channelwidth = numpy.array( self.freqwin * [self.frequency[1] - self.frequency[0]]) else: self.channelwidth = numpy.array([1e6]) if dopol: self.vis_pol = PolarisationFrame('linear') self.image_pol = PolarisationFrame('stokesIQUV') f = numpy.array([100.0, 20.0, -10.0, 1.0]) else: self.vis_pol = PolarisationFrame('stokesI') self.image_pol = PolarisationFrame('stokesI') f = numpy.array([100.0]) if dospectral: flux = numpy.array( [f * numpy.power(freq / 1e8, -0.7) for freq in self.frequency]) else: flux = numpy.array([f]) self.phasecentre = SkyCoord(ra=+180.0 * u.deg, dec=-60.0 * u.deg, frame='icrs', equinox='J2000') self.blockvis_list = [ ingest_unittest_visibility(self.low, [self.frequency[i]], [self.channelwidth[i]], self.times, self.vis_pol, self.phasecentre, block=True, zerow=zerow) for i in range(nfreqwin) ] for v in self.blockvis_list: v.data['vis'][...] = 1.0 + 0.0j self.error_blockvis_list = [ copy_visibility(v) for v in self.blockvis_list ] gt = create_gaintable_from_blockvisibility(self.blockvis_list[0]) gt = simulate_gaintable(gt, phase_error=0.1, amplitude_error=0.0, smooth_channels=1, leakage=0.0) self.error_blockvis_list = [ apply_gaintable(self.error_blockvis_list[i], gt) for i in range(self.freqwin) ] assert numpy.max( numpy.abs(self.error_blockvis_list[0].vis - self.blockvis_list[0].vis)) > 0.0
def actualSetUp(self, add_errors=False, nfreqwin=7, dospectral=True, dopol=False, zerow=True): self.npixel = 512 self.low = create_named_configuration('LOWBD2', rmax=750.0) self.freqwin = nfreqwin self.vis_list = list() self.ntimes = 5 self.times = numpy.linspace(-3.0, +3.0, self.ntimes) * numpy.pi / 12.0 self.frequency = numpy.linspace(0.8e8, 1.2e8, self.freqwin) if self.freqwin > 1: self.channelwidth = numpy.array( self.freqwin * [self.frequency[1] - self.frequency[0]]) else: self.channelwidth = numpy.array([1e6]) if dopol: self.vis_pol = PolarisationFrame('linear') self.image_pol = PolarisationFrame('stokesIQUV') f = numpy.array([100.0, 20.0, 0.0, 0.0]) else: self.vis_pol = PolarisationFrame('stokesI') self.image_pol = PolarisationFrame('stokesI') f = numpy.array([100.0]) if dospectral: flux = numpy.array( [f * numpy.power(freq / 1e8, -0.7) for freq in self.frequency]) else: flux = numpy.array([f]) self.phasecentre = SkyCoord(ra=+180.0 * u.deg, dec=-60.0 * u.deg, frame='icrs', equinox='J2000') self.blockvis_list = \ [rsexecute.execute(ingest_unittest_visibility, nout=1)(self.low, [self.frequency[i]], [self.channelwidth[i]], self.times, self.vis_pol, self.phasecentre, block=True, zerow=zerow) for i in range(nfreqwin)] self.blockvis_list = rsexecute.compute(self.blockvis_list, sync=True) self.blockvis_list = rsexecute.scatter(self.blockvis_list) self.vis_list = [ rsexecute.execute(convert_blockvisibility_to_visibility, nout=1)(bv) for bv in self.blockvis_list ] self.vis_list = rsexecute.compute(self.vis_list, sync=True) self.vis_list = rsexecute.scatter(self.vis_list) self.model_imagelist = [ rsexecute.execute(create_unittest_model, nout=1)(self.vis_list[i], self.image_pol, npixel=self.npixel, cellsize=0.0005) for i in range(nfreqwin) ] self.model_imagelist = rsexecute.compute(self.model_imagelist, sync=True) self.model_imagelist = rsexecute.scatter(self.model_imagelist) self.components_list = [ rsexecute.execute(create_unittest_components)( self.model_imagelist[freqwin], flux[freqwin, :][numpy.newaxis, :]) for freqwin, m in enumerate(self.model_imagelist) ] self.components_list = rsexecute.compute(self.components_list, sync=True) self.components_list = rsexecute.scatter(self.components_list) self.blockvis_list = [ rsexecute.execute(dft_skycomponent_visibility)( self.blockvis_list[freqwin], self.components_list[freqwin]) for freqwin, _ in enumerate(self.blockvis_list) ] self.blockvis_list = rsexecute.compute(self.blockvis_list, sync=True) self.vis = self.blockvis_list[0] self.blockvis_list = rsexecute.scatter(self.blockvis_list) self.model_imagelist = [ rsexecute.execute(insert_skycomponent, nout=1)(self.model_imagelist[freqwin], self.components_list[freqwin]) for freqwin in range(nfreqwin) ] self.model_imagelist = rsexecute.compute(self.model_imagelist, sync=True) model = self.model_imagelist[0] self.cmodel = smooth_image(model) if self.persist: export_image_to_fits( model, '%s/test_pipelines_rsexecute_model.fits' % self.dir) export_image_to_fits( self.cmodel, '%s/test_pipelines_rsexecute_cmodel.fits' % self.dir) if add_errors: gt = create_gaintable_from_blockvisibility(self.vis) gt = simulate_gaintable(gt, phase_error=0.1, amplitude_error=0.0, smooth_channels=1, leakage=0.0) self.blockvis_list = [ rsexecute.execute(apply_gaintable, nout=1)(self.blockvis_list[i], gt) for i in range(self.freqwin) ] self.blockvis_list = rsexecute.compute(self.blockvis_list, sync=True) self.blockvis_list = rsexecute.scatter(self.blockvis_list) self.vis_list = [ rsexecute.execute(convert_blockvisibility_to_visibility)(bv) for bv in self.blockvis_list ] self.vis_list = rsexecute.compute(self.vis_list, sync=True) self.vis_list = rsexecute.scatter(self.vis_list) self.model_imagelist = [ rsexecute.execute(create_unittest_model, nout=1)(self.vis_list[i], self.image_pol, npixel=self.npixel, cellsize=0.0005) for i in range(nfreqwin) ] self.model_imagelist = rsexecute.compute(self.model_imagelist, sync=True) self.model_imagelist = rsexecute.scatter(self.model_imagelist)
def ingest_visibility(self, freq=None, chan_width=None, times=None, add_errors=False, block=True, bandpass=False): if freq is None: freq = [1e8] if chan_width is None: chan_width = [1e6] if times is None: times = (numpy.pi / 12.0) * numpy.linspace(-3.0, 3.0, 5) lowcore = create_named_configuration('LOWBD2', rmax=750.0) frequency = numpy.array(freq) channel_bandwidth = numpy.array(chan_width) phasecentre = SkyCoord(ra=+180.0 * u.deg, dec=-60.0 * u.deg, frame='icrs', equinox='J2000') if block: vt = create_blockvisibility( lowcore, times, frequency, channel_bandwidth=channel_bandwidth, weight=1.0, phasecentre=phasecentre, polarisation_frame=PolarisationFrame("stokesI")) else: vt = create_visibility( lowcore, times, frequency, channel_bandwidth=channel_bandwidth, weight=1.0, phasecentre=phasecentre, polarisation_frame=PolarisationFrame("stokesI")) cellsize = 0.001 model = create_image_from_visibility( vt, npixel=self.npixel, cellsize=cellsize, npol=1, frequency=frequency, phasecentre=phasecentre, polarisation_frame=PolarisationFrame("stokesI")) nchan = len(self.frequency) flux = numpy.array(nchan * [[100.0]]) facets = 4 rpix = model.wcs.wcs.crpix - 1.0 spacing_pixels = self.npixel // facets centers = [-1.5, -0.5, 0.5, 1.5] comps = list() for iy in centers: for ix in centers: p = int(round(rpix[0] + ix * spacing_pixels * numpy.sign(model.wcs.wcs.cdelt[0]))), \ int(round(rpix[1] + iy * spacing_pixels * numpy.sign(model.wcs.wcs.cdelt[1]))) sc = pixel_to_skycoord(p[0], p[1], model.wcs, origin=1) comp = create_skycomponent( direction=sc, flux=flux, frequency=frequency, polarisation_frame=PolarisationFrame("stokesI")) comps.append(comp) if block: dft_skycomponent_visibility(vt, comps) else: dft_skycomponent_visibility(vt, comps) insert_skycomponent(model, comps) self.comps = comps self.model = copy_image(model) self.empty_model = create_empty_image_like(model) export_image_to_fits( model, '%s/test_pipeline_functions_model.fits' % (self.dir)) if add_errors: # These will be the same for all calls numpy.random.seed(180555) gt = create_gaintable_from_blockvisibility(vt) gt = simulate_gaintable(gt, phase_error=1.0, amplitude_error=0.0) vt = apply_gaintable(vt, gt) if bandpass: bgt = create_gaintable_from_blockvisibility(vt, timeslice=1e5) bgt = simulate_gaintable(bgt, phase_error=0.01, amplitude_error=0.01, smooth_channels=4) vt = apply_gaintable(vt, bgt) return vt