def core_solve(self, spf, dpf, phase_error=0.1, amplitude_error=0.0, leakage=0.0, phase_only=True, niter=200, crosspol=False, residual_tol=1e-6, f=None, vnchan=3): if f is None: f = [100.0, 50.0, -10.0, 40.0] self.actualSetup(spf, dpf, f=f, vnchan=vnchan) gt = create_gaintable_from_blockvisibility(self.vis) log.info("Created gain table: %s" % (gaintable_summary(gt))) gt = simulate_gaintable(gt, phase_error=phase_error, amplitude_error=amplitude_error, leakage=leakage) original = copy_visibility(self.vis) vis = apply_gaintable(self.vis, gt) gtsol = solve_gaintable(self.vis, original, phase_only=phase_only, niter=niter, crosspol=crosspol, tol=1e-6) vis = apply_gaintable(vis, gtsol, inverse=True) residual = numpy.max(gtsol.residual) assert residual < residual_tol, "%s %s Max residual = %s" % (spf, dpf, residual) log.debug(qa_gaintable(gt)) assert numpy.max(numpy.abs(gtsol.gain - 1.0)) > 0.1
def test_apply_gaintable_and_inverse_both(self): for spf, dpf in[('stokesI', 'stokesI'), ('stokesIQUV', 'linear'), ('stokesIQUV', 'circular')]: self.actualSetup(spf, dpf) gt = create_gaintable_from_blockvisibility(self.vis, timeslice='auto') log.info("Created gain table: %s" % (gaintable_summary(gt))) gt = simulate_gaintable(gt, phase_error=0.1, amplitude_error=0.1) original = copy_visibility(self.vis) vis = apply_gaintable(self.vis, gt) vis = apply_gaintable(self.vis, gt, inverse=True) error = numpy.max(numpy.abs(vis.vis - original.vis)) assert error < 1e-12, "Error = %s" % (error)
def insert_unittest_errors(vt, seed=180555, amp_errors=None, phase_errors=None): """Simulate gain errors and apply :param vt: :param seed: Random number seed, set to big integer repeat values from run to run :param phase_errors: e.g. {'T': 1.0, 'G': 0.1, 'B': 0.01} :param amp_errors: e.g. {'T': 0.0, 'G': 0.01, 'B': 0.01} :return: """ numpy.random.seed(seed) controls = create_calibration_controls() if amp_errors is None: amp_errors = {'T': 0.0, 'G': 0.01, 'B': 0.01} if phase_errors is None: phase_errors = {'T': 1.0, 'G': 0.1, 'B': 0.01} for c in "TGB": gaintable = \ create_gaintable_from_blockvisibility(vt, timeslice=controls[c]['timeslice']) gaintable = simulate_gaintable(gaintable, timeslice=controls[c]['timeslice'], phase_only=controls[c]['phase_only'], crosspol=controls[c]['shape'] == 'matrix', phase_error=phase_errors[c], amplitude_error=amp_errors[c]) vt = apply_gaintable(vt, gaintable, inverse=True, timeslice=controls[c]['timeslice']) return vt
def test_apply_gaintable_null(self): for spf, dpf in[('stokesI', 'stokesI'), ('stokesIQUV', 'linear'), ('stokesIQUV', 'circular')]: self.actualSetup(spf, dpf) gt = create_gaintable_from_blockvisibility(self.vis, timeslice='auto') gt.data['gain']*=0.0 original = copy_visibility(self.vis) vis = apply_gaintable(self.vis, gt, inverse=True) error = numpy.max(numpy.abs(vis.vis[:,0,1,...] - original.vis[:,0,1,...])) assert error < 1e-12, "Error = %s" % (error)
def create_blockvisibility_iterator(config: Configuration, times: numpy.array, frequency: numpy.array, channel_bandwidth, phasecentre: SkyCoord, weight: float = 1, polarisation_frame=PolarisationFrame('stokesI'), integration_time=1.0, number_integrations=1, predict=predict_2d, model=None, components=None, phase_error=0.0, amplitude_error=0.0, sleep=0.0, **kwargs): """ Create a sequence of Visibilities and optionally predicting and coalescing This is useful mainly for performing large simulations. Do something like:: vis_iter = create_blockvisibility_iterator(config, times, frequency, channel_bandwidth, phasecentre=phasecentre, weight=1.0, integration_time=30.0, number_integrations=3) for i, vis in enumerate(vis_iter): if i == 0: fullvis = vis else: fullvis = append_visibility(fullvis, vis) :param config: Configuration of antennas :param times: hour angles in radians :param frequency: frequencies (Hz] Shape [nchan] :param weight: weight of a single sample :param phasecentre: phasecentre of observation :param npol: Number of polarizations :param integration_time: Integration time ('auto' or value in s) :param number_integrations: Number of integrations to be created at each time. :param model: Model image to be inserted :param components: Components to be inserted :param sleep_time: Time to sleep between yields :return: Visibility """ for time in times: actualtimes = time + numpy.arange(0, number_integrations) * integration_time * numpy.pi / 43200.0 bvis = create_blockvisibility(config, actualtimes, frequency=frequency, phasecentre=phasecentre, weight=weight, polarisation_frame=polarisation_frame, integration_time=integration_time, channel_bandwidth=channel_bandwidth) if model is not None: vis = predict(bvis, model, **kwargs) bvis = convert_visibility_to_blockvisibility(vis) if components is not None: bvis = predict_skycomponent_visibility(bvis, components) # Add phase errors if phase_error > 0.0 or amplitude_error > 0.0: gt = create_gaintable_from_blockvisibility(bvis) gt = simulate_gaintable(gt=gt, phase_error=phase_error, amplitude_error=amplitude_error) bvis = apply_gaintable(bvis, gt) import time time.sleep(sleep) yield bvis
def make_e(vis, calskymodel, evis_all): # Return the estep for a given skymodel evis = copy_visibility(vis) tvis = copy_visibility(vis, zero=True) tvis = predict_skymodel_visibility(tvis, calskymodel[0]) tvis = apply_gaintable(tvis, calskymodel[1]) # E step is the data model for a window plus the difference between the observed data_models # and the summed data models or, put another way, its the observed data minus the # summed visibility for all other windows evis.data['vis'][...] = tvis.data['vis'][...] + vis.data['vis'][...] - evis_all.data['vis'][...] return evis
def test_create_gaintable_from_visibility(self): for spf, dpf in[('stokesI', 'stokesI'), ('stokesIQUV', 'linear'), ('stokesIQUV', 'circular')]: self.actualSetup(spf, dpf) gt = create_gaintable_from_blockvisibility(self.vis, timeslice='auto') log.info("Created gain table: %s" % (gaintable_summary(gt))) gt = simulate_gaintable(gt, phase_error=1.0) original = copy_visibility(self.vis) vis = apply_gaintable(self.vis, gt) assert numpy.max(numpy.abs(original.vis)) > 0.0 assert numpy.max(numpy.abs(vis.vis)) > 0.0 assert numpy.max(numpy.abs(vis.vis - original.vis)) > 0.0
def test_solve_gaintable_scalar_bandpass(self): self.actualSetup('stokesI', 'stokesI', f=[100.0], vnchan=128) gt = create_gaintable_from_blockvisibility(self.vis) log.info("Created gain table: %s" % (gaintable_summary(gt))) gt = simulate_gaintable(gt, phase_error=10.0, amplitude_error=0.01, smooth_channels=8) original = copy_visibility(self.vis) self.vis = apply_gaintable(self.vis, gt) gtsol = solve_gaintable(self.vis, original, phase_only=False, niter=200) residual = numpy.max(gtsol.residual) assert residual < 3e-8, "Max residual = %s" % (residual) assert numpy.max(numpy.abs(gtsol.gain - 1.0)) > 0.1
def test_solve_gaintable_scalar_timeslice(self): self.actualSetup('stokesI', 'stokesI', f=[100.0], ntimes=10) gt = create_gaintable_from_blockvisibility(self.vis, timeslice=120.0) log.info("Created gain table: %s" % (gaintable_summary(gt))) gt = simulate_gaintable(gt, phase_error=10.0, amplitude_error=0.0) original = copy_visibility(self.vis) self.vis = apply_gaintable(self.vis, gt) gtsol = solve_gaintable(self.vis, original, phase_only=True, niter=200) residual = numpy.max(gtsol.residual) assert residual < 3e-8, "Max residual = %s" % (residual) assert numpy.max(numpy.abs(gtsol.gain - 1.0)) > 0.1
def test_solve_gaintable_scalar_normalise(self): self.actualSetup('stokesI', 'stokesI', f=[100.0]) gt = create_gaintable_from_blockvisibility(self.vis) log.info("Created gain table: %s" % (gaintable_summary(gt))) gt = simulate_gaintable(gt, phase_error=0.0, amplitude_error=0.1) gt.data['gain'] *= 2.0 original = copy_visibility(self.vis) self.vis = apply_gaintable(self.vis, gt) gtsol = solve_gaintable(self.vis, original, phase_only=False, niter=200, normalise_gains=True) residual = numpy.max(gtsol.residual) assert residual < 3e-8, "Max residual = %s" % (residual) assert numpy.max(numpy.abs(gtsol.gain - 1.0)) > 0.1
def modelpartition_list_expectation_step(vis: BlockVisibility, evis_all: BlockVisibility, modelpartition, **kwargs): """Calculates E step in equation A12 This is the data model for this window plus the difference between observed data and summed data models :param evis_all: Sum data models :param csm: csm element being fit :param kwargs: :return: Data model (i.e. visibility) for this csm """ evis = copy_visibility(evis_all) tvis = copy_visibility(vis, zero=True) tvis = predict_skymodel_visibility(tvis, modelpartition[0], **kwargs) tvis = apply_gaintable(tvis, modelpartition[1]) evis.data['vis'][...] = tvis.data['vis'][...] + vis.data['vis'][...] - evis_all.data['vis'][...] return evis
def calskymodel_expectation_all(vis: BlockVisibility, calskymodels, **kwargs): """Calculates E step in equation A12 This is the sum of the data models over all skymodel :param vis: Visibility :param csm: List of (skymodel, gaintable) tuples :param kwargs: :return: Sum of data models (i.e. a visibility) """ evis = copy_visibility(vis, zero=True) tvis = copy_visibility(vis, zero=True) for csm in calskymodels: tvis.data['vis'][...] = 0.0 tvis = predict_skymodel_visibility(tvis, csm[0], **kwargs) tvis = apply_gaintable(tvis, csm[1]) evis.data['vis'][...] += tvis.data['vis'][...] return evis
def calibrate_list_serial_workflow(vis_list, model_vislist, calibration_context='TG', global_solution=True, **kwargs): """ Create a set of components for (optionally global) calibration of a list of visibilities If global solution is true then visibilities are gathered to a single visibility data set which is then self-calibrated. The resulting gaintable is then effectively scattered out for application to each visibility set. If global solution is false then the solutions are performed locally. :param vis_list: :param model_vislist: :param calibration_context: String giving terms to be calibrated e.g. 'TGB' :param global_solution: Solve for global gains :param kwargs: Parameters for functions in components :return: """ def solve_and_apply(vis, modelvis=None): return calibrate_function(vis, modelvis, calibration_context=calibration_context, **kwargs)[0] if global_solution: point_vislist = [ divide_visibility(vis_list[i], model_vislist[i]) for i, _ in enumerate(vis_list) ] global_point_vis_list = visibility_gather_channel(point_vislist) global_point_vis_list = integrate_visibility_by_channel( global_point_vis_list) # This is a global solution so we only compute one gain table _, gt_list = solve_and_apply(global_point_vis_list) return [apply_gaintable(v, gt_list, inverse=True) for v in vis_list] else: return [ solve_and_apply(vis_list[i], model_vislist[i]) for i, v in enumerate(vis_list) ]
def ingest_visibility(self, freq=None, chan_width=None, times=None, add_errors=False, block=True, bandpass=False): if freq is None: freq = [1e8] if chan_width is None: chan_width = [1e6] if times is None: times = (numpy.pi / 12.0) * numpy.linspace(-3.0, 3.0, 5) lowcore = create_named_configuration('LOWBD2', rmax=750.0) frequency = numpy.array(freq) channel_bandwidth = numpy.array(chan_width) phasecentre = SkyCoord(ra=+180.0 * u.deg, dec=-60.0 * u.deg, frame='icrs', equinox='J2000') if block: vt = create_blockvisibility( lowcore, times, frequency, channel_bandwidth=channel_bandwidth, weight=1.0, phasecentre=phasecentre, polarisation_frame=PolarisationFrame("stokesI")) else: vt = create_visibility( lowcore, times, frequency, channel_bandwidth=channel_bandwidth, weight=1.0, phasecentre=phasecentre, polarisation_frame=PolarisationFrame("stokesI")) cellsize = 0.001 model = create_image_from_visibility( vt, npixel=self.npixel, cellsize=cellsize, npol=1, frequency=frequency, phasecentre=phasecentre, polarisation_frame=PolarisationFrame("stokesI")) nchan = len(self.frequency) flux = numpy.array(nchan * [[100.0]]) facets = 4 rpix = model.wcs.wcs.crpix - 1.0 spacing_pixels = self.npixel // facets centers = [-1.5, -0.5, 0.5, 1.5] comps = list() for iy in centers: for ix in centers: p = int(round(rpix[0] + ix * spacing_pixels * numpy.sign(model.wcs.wcs.cdelt[0]))), \ int(round(rpix[1] + iy * spacing_pixels * numpy.sign(model.wcs.wcs.cdelt[1]))) sc = pixel_to_skycoord(p[0], p[1], model.wcs, origin=1) comp = create_skycomponent( direction=sc, flux=flux, frequency=frequency, polarisation_frame=PolarisationFrame("stokesI")) comps.append(comp) if block: predict_skycomponent_visibility(vt, comps) else: predict_skycomponent_visibility(vt, comps) insert_skycomponent(model, comps) self.comps = comps self.model = copy_image(model) self.empty_model = create_empty_image_like(model) export_image_to_fits( model, '%s/test_pipeline_functions_model.fits' % (self.dir)) if add_errors: # These will be the same for all calls numpy.random.seed(180555) gt = create_gaintable_from_blockvisibility(vt) gt = simulate_gaintable(gt, phase_error=1.0, amplitude_error=0.0) vt = apply_gaintable(vt, gt) if bandpass: bgt = create_gaintable_from_blockvisibility(vt, timeslice=1e5) bgt = simulate_gaintable(bgt, phase_error=0.01, amplitude_error=0.01, smooth_channels=4) vt = apply_gaintable(vt, bgt) return vt
def actualSetup(self, vnchan=1, doiso=True, ntimes=5, flux_limit=2.0, zerow=True, fixed=False): nfreqwin = vnchan rmax = 300.0 npixel = 512 cellsize = 0.001 frequency = numpy.linspace(0.8e8, 1.2e8, nfreqwin) if nfreqwin > 1: channel_bandwidth = numpy.array(nfreqwin * [frequency[1] - frequency[0]]) else: channel_bandwidth = [0.4e8] times = numpy.linspace(-numpy.pi / 3.0, numpy.pi / 3.0, ntimes) phasecentre = SkyCoord(ra=-60.0 * u.deg, dec=-60.0 * u.deg, frame='icrs', equinox='J2000') lowcore = create_named_configuration('LOWBD2', rmax=rmax) block_vis = create_blockvisibility( lowcore, times, frequency=frequency, channel_bandwidth=channel_bandwidth, weight=1.0, phasecentre=phasecentre, polarisation_frame=PolarisationFrame("stokesI"), zerow=zerow) block_vis.data['uvw'][..., 2] = 0.0 self.beam = create_image_from_visibility( block_vis, npixel=npixel, frequency=[numpy.average(frequency)], nchan=nfreqwin, channel_bandwidth=[numpy.sum(channel_bandwidth)], cellsize=cellsize, phasecentre=phasecentre) self.components = create_low_test_skycomponents_from_gleam( flux_limit=flux_limit, phasecentre=phasecentre, frequency=frequency, polarisation_frame=PolarisationFrame('stokesI'), radius=npixel * cellsize) self.beam = create_low_test_beam(self.beam) self.components = apply_beam_to_skycomponent(self.components, self.beam, flux_limit=flux_limit) self.vis = copy_visibility(block_vis, zero=True) gt = create_gaintable_from_blockvisibility(block_vis, timeslice='auto') for i, sc in enumerate(self.components): if sc.flux[0, 0] > 10: sc.flux[...] /= 10.0 component_vis = copy_visibility(block_vis, zero=True) gt = simulate_gaintable(gt, amplitude_error=0.0, phase_error=0.1, seed=None) component_vis = predict_skycomponent_visibility(component_vis, sc) component_vis = apply_gaintable(component_vis, gt) self.vis.data['vis'][...] += component_vis.data['vis'][...] # Do an isoplanatic selfcal self.model_vis = copy_visibility(self.vis, zero=True) self.model_vis = predict_skycomponent_visibility( self.model_vis, self.components) if doiso: gt = solve_gaintable(self.vis, self.model_vis, phase_only=True, timeslice='auto') self.vis = apply_gaintable(self.vis, gt, inverse=True) self.model_vis = convert_blockvisibility_to_visibility(self.model_vis) self.model_vis, _, _ = weight_visibility(self.model_vis, self.beam) self.dirty_model, sumwt = invert_function(self.model_vis, self.beam, context='2d') export_image_to_fits(self.dirty_model, "%s/test_skymodel-model_dirty.fits" % self.dir) lvis = convert_blockvisibility_to_visibility(self.vis) lvis, _, _ = weight_visibility(lvis, self.beam) dirty, sumwt = invert_function(lvis, self.beam, context='2d') if doiso: export_image_to_fits( dirty, "%s/test_skymodel-initial-iso-residual.fits" % self.dir) else: export_image_to_fits( dirty, "%s/test_skymodel-initial-noiso-residual.fits" % self.dir) self.skymodels = [ SkyModel(components=[cm], fixed=fixed) for cm in self.components ]
block_vis = convert_visibility_to_blockvisibility(predicted_vis) #print("np.sum(block_vis.data): ", numpy.sum(block_vis.data['vis'])) #print("nchan npol nants ", block_vis.nchan, block_vis.npol, block_vis.nants) #print("uvw", block_vis.uvw, numpy.sum(block_vis.uvw)) #print("vis", block_vis.vis, numpy.sum(block_vis.vis)) #print("weight", block_vis.weight, numpy.sum(block_vis.weight)) #print("time", block_vis.time, numpy.sum(block_vis.time)) #print("integration_time", block_vis.integration_time, numpy.sum(block_vis.integration_time)) #print("nvis, size", block_vis.nvis, block_vis.size()) gt = create_gaintable_from_blockvisibility(block_vis) #print("np.sum(gt.data): ", numpy.sum(gt.data['gain'])) gt = simulate_gaintable(gt, phase_error=1.0) #print("np.sum(gt.data): ", numpy.sum(gt.data['gain'])) blockvis = apply_gaintable(block_vis, gt) #print("np.sum(blockvis.data): ", numpy.sum(blockvis.data['vis'])) model = create_image_from_visibility( block_vis, npixel=npixel, frequency=[numpy.average(frequency)], nchan=1, channel_bandwidth=[numpy.sum(channel_bandwidth)], cellsize=cellsize, phasecentre=phasecentre) #print("model sum, min, max, shape: ", numpy.sum(model.data), numpy.amin(model.data), numpy.amax(model.data), model.shape) print(qa_image(model, context='Blockvis model image')) export_image_to_fits(model, '%s/imaging-blockvis_model.fits' % (results_dir))
def predict_and_apply(ovis, calskymodel): tvis = copy_visibility(ovis, zero=True) tvis = predict_skymodel_visibility(tvis, calskymodel[0]) tvis = apply_gaintable(tvis, calskymodel[1]) return tvis
def corrupt_vis(vis, gt, **kwargs): if gt is None: gt = create_gaintable_from_blockvisibility(vis, **kwargs) gt = simulate_gaintable(gt, **kwargs) return apply_gaintable(vis, gt)
def predict_and_apply(ovis, modelpartition): tvis = copy_visibility(ovis, zero=True) tvis = predict_skymodel_visibility(tvis, modelpartition[0]) tvis = apply_gaintable(tvis, modelpartition[1]) return tvis