def solve_image_arlexecute(vis: Visibility, model: Image, components=None, context='2d', **kwargs) -> \
(Visibility, Image, Image):
"""Solve for image using deconvolve_cube and specified predict, invert
This is the same as a majorcycle/minorcycle algorithm. The components are removed prior to deconvolution.
See also arguments for predict, invert, deconvolve_cube functions.2d
:param vis:
:param model: Model image
:param predict: Predict function e.g. predict_2d, predict_wstack
:param invert: Invert function e.g. invert_2d, invert_wstack
:return: Visibility, model
"""
nmajor = get_parameter(kwargs, 'nmajor', 5)
thresh = get_parameter(kwargs, "threshold", 0.0)
log.info("solve_image_arlexecute: Performing %d major cycles" % nmajor)
# The model is added to each major cycle and then the visibilities are
# calculated from the full model
vispred = copy_visibility(vis, zero=True)
visres = copy_visibility(vis, zero=True)
vispred = predict_arlexecute(vispred, model, context=context, **kwargs)
if components is not None:
vispred = predict_skycomponent_visibility(vispred, components)
visres.data['vis'] = vis.data['vis'] - vispred.data['vis']
dirty, sumwt = invert_arlexecute(visres, model, context=context, dopsf=False, **kwargs)
assert sumwt.any() > 0.0, "Sum of weights is zero"
psf, sumwt = invert_arlexecute(visres, model, context=context, dopsf=True, **kwargs)
assert sumwt.any() > 0.0, "Sum of weights is zero"
for i in range(nmajor):
log.info("solve_image_arlexecute: Start of major cycle %d" % i)
cc, res = deconvolve_cube(dirty, psf, **kwargs)
model.data += cc.data
vispred.data['vis'][...]=0.0
vispred = predict_arlexecute(vispred, model, context=context, **kwargs)
visres.data['vis'] = vis.data['vis'] - vispred.data['vis']
dirty, sumwt = invert_arlexecute(visres, model, context=context, dopsf=False, **kwargs)
if numpy.abs(dirty.data).max() < 1.1 * thresh:
log.info("Reached stopping threshold %.6f Jy" % thresh)
break
log.info("solve_image_arlexecute: End of minor cycles")
log.info("solve_image_arlexecute: End of major cycles")
return visres, model, dirty
def _predict_base(self, fluxthreshold=1.0, name='predict_ng', **kwargs):
from processing_components.imaging.ng import predict_ng, invert_ng
original_vis = copy_visibility(self.blockvis)
vis = predict_ng(self.blockvis,
self.model,
verbosity=self.verbosity,
**kwargs)
vis.data['vis'] = vis.data['vis'] - original_vis.data['vis']
dirty = invert_ng(vis,
self.model,
dopsf=False,
normalize=True,
verbosity=self.verbosity,
**kwargs)
# import matplotlib.pyplot as plt
# from processing_components.image.operations import show_image
# npol = dirty[0].shape[1]
# for pol in range(npol):
# plt.clf()
# show_image(dirty[0], pol=pol)
# plt.show(block=False)
if self.persist:
export_image_to_fits(
dirty[0],
'%s/test_imaging_ng_%s_residual.fits' % (self.dir, name))
# assert numpy.max(numpy.abs(dirty[0].data)), "Residual image is empty"
maxabs = numpy.max(numpy.abs(dirty[0].data))
assert maxabs < fluxthreshold, "Error %.3f greater than fluxthreshold %.3f " % (
maxabs, fluxthreshold)
def core_solve(self,
spf,
dpf,
phase_error=0.1,
amplitude_error=0.0,
leakage=0.0,
phase_only=True,
niter=200,
crosspol=False,
residual_tol=1e-6,
f=None,
vnchan=3):
if f is None:
f = [100.0, 50.0, -10.0, 40.0]
self.actualSetup(spf, dpf, f=f, vnchan=vnchan)
gt = create_gaintable_from_blockvisibility(self.vis)
log.info("Created gain table: %s" % (gaintable_summary(gt)))
gt = simulate_gaintable(gt,
phase_error=phase_error,
amplitude_error=amplitude_error,
leakage=leakage)
original = copy_visibility(self.vis)
vis = apply_gaintable(self.vis, gt)
gtsol = solve_gaintable(self.vis,
original,
phase_only=phase_only,
niter=niter,
crosspol=crosspol,
tol=1e-6)
vis = apply_gaintable(vis, gtsol, inverse=True)
residual = numpy.max(gtsol.residual)
assert residual < residual_tol, "%s %s Max residual = %s" % (spf, dpf,
residual)
log.debug(qa_gaintable(gt))
assert numpy.max(numpy.abs(gtsol.gain - 1.0)) > 0.1
def zero(vis):
if vis is not None:
zerovis = copy_visibility(vis)
zerovis.data['vis'][...] = 0.0
return zerovis
else:
return None
def subtract_vis(vis, model_vis):
if vis is not None and model_vis is not None:
assert vis.vis.shape == model_vis.vis.shape
subvis = copy_visibility(vis)
subvis.data['vis'][...] -= model_vis.data['vis'][...]
return subvis
else:
return None
def test_copy_visibility(self):
self.vis = create_visibility(self.lowcore, self.times, self.frequency,
channel_bandwidth=self.channel_bandwidth, phasecentre=self.phasecentre, weight=1.0,
polarisation_frame=PolarisationFrame("stokesIQUV"))
vis = copy_visibility(self.vis)
self.vis.data['vis'] = 0.0
vis.data['vis'] = 1.0
assert (vis.data['vis'][0, 0].real == 1.0)
assert (self.vis.data['vis'][0, 0].real == 0.0)
def test_apply_gaintable_null(self):
for spf, dpf in[('stokesI', 'stokesI'), ('stokesIQUV', 'linear'), ('stokesIQUV', 'circular')]:
self.actualSetup(spf, dpf)
gt = create_gaintable_from_blockvisibility(self.vis, timeslice='auto')
gt.data['gain']*=0.0
original = copy_visibility(self.vis)
vis = apply_gaintable(self.vis, gt, inverse=True)
error = numpy.max(numpy.abs(vis.vis[:,0,1,...] - original.vis[:,0,1,...]))
assert error < 1e-12, "Error = %s" % (error)
def test_apply_gaintable_only(self):
for spf, dpf in[('stokesI', 'stokesI'), ('stokesIQUV', 'linear'), ('stokesIQUV', 'circular')]:
self.actualSetup(spf, dpf)
gt = create_gaintable_from_blockvisibility(self.vis, timeslice='auto')
log.info("Created gain table: %s" % (gaintable_summary(gt)))
gt = simulate_gaintable(gt, phase_error=0.1, amplitude_error=0.01)
original = copy_visibility(self.vis)
vis = apply_gaintable(self.vis, gt)
error = numpy.max(numpy.abs(vis.vis - original.vis))
assert error > 10.0, "Error = %f" % (error)
def test_solve_gaintable_scalar_bandpass(self):
self.actualSetup('stokesI', 'stokesI', f=[100.0], vnchan=128)
gt = create_gaintable_from_blockvisibility(self.vis)
log.info("Created gain table: %s" % (gaintable_summary(gt)))
gt = simulate_gaintable(gt, phase_error=10.0, amplitude_error=0.01, smooth_channels=8)
original = copy_visibility(self.vis)
self.vis = apply_gaintable(self.vis, gt)
gtsol = solve_gaintable(self.vis, original, phase_only=False, niter=200)
residual = numpy.max(gtsol.residual)
assert residual < 3e-8, "Max residual = %s" % (residual)
assert numpy.max(numpy.abs(gtsol.gain - 1.0)) > 0.1
def test_solve_gaintable_scalar_timeslice(self):
self.actualSetup('stokesI', 'stokesI', f=[100.0], ntimes=10)
gt = create_gaintable_from_blockvisibility(self.vis, timeslice=120.0)
log.info("Created gain table: %s" % (gaintable_summary(gt)))
gt = simulate_gaintable(gt, phase_error=10.0, amplitude_error=0.0)
original = copy_visibility(self.vis)
self.vis = apply_gaintable(self.vis, gt)
gtsol = solve_gaintable(self.vis, original, phase_only=True, niter=200)
residual = numpy.max(gtsol.residual)
assert residual < 3e-8, "Max residual = %s" % (residual)
assert numpy.max(numpy.abs(gtsol.gain - 1.0)) > 0.1
def test_solve_gaintable_scalar_normalise(self):
self.actualSetup('stokesI', 'stokesI', f=[100.0])
gt = create_gaintable_from_blockvisibility(self.vis)
log.info("Created gain table: %s" % (gaintable_summary(gt)))
gt = simulate_gaintable(gt, phase_error=0.0, amplitude_error=0.1)
gt.data['gain'] *= 2.0
original = copy_visibility(self.vis)
self.vis = apply_gaintable(self.vis, gt)
gtsol = solve_gaintable(self.vis, original, phase_only=False, niter=200, normalise_gains=True)
residual = numpy.max(gtsol.residual)
assert residual < 3e-8, "Max residual = %s" % (residual)
assert numpy.max(numpy.abs(gtsol.gain - 1.0)) > 0.1
def test_create_gaintable_from_visibility_interval(self):
for timeslice in [10.0, 'auto', 1e5]:
for spf, dpf in[('stokesIQUV', 'linear')]:
self.actualSetup(spf, dpf)
gt = create_gaintable_from_blockvisibility(self.vis, timeslice=timeslice)
log.info("Created gain table: %s" % (gaintable_summary(gt)))
gt = simulate_gaintable(gt, phase_error=1.0)
original = copy_visibility(self.vis)
vis = apply_gaintable(self.vis, gt)
assert numpy.max(numpy.abs(original.vis)) > 0.0
assert numpy.max(numpy.abs(vis.vis)) > 0.0
assert numpy.max(numpy.abs(vis.vis - original.vis)) > 0.0
def test_addnoise_blockvisibility(self):
self.vis = create_blockvisibility(
self.config,
self.times,
self.frequency,
phasecentre=self.phasecentre,
weight=1.0,
polarisation_frame=PolarisationFrame('stokesIQUV'),
channel_bandwidth=self.channel_bandwidth)
original = copy_visibility(self.vis)
self.vis = addnoise_visibility(self.vis)
actual = numpy.std(numpy.abs(self.vis.vis - original.vis))
assert abs(actual - 0.01077958403015586) < 1e-4, actual
def test_calibrate_T_function(self):
self.actualSetup('stokesI', 'stokesI', f=[100.0])
# Prepare the corrupted visibility data_models
gt = create_gaintable_from_blockvisibility(self.vis)
log.info("Created gain table: %s" % (gaintable_summary(gt)))
gt = simulate_gaintable(gt, phase_error=10.0, amplitude_error=0.0)
original = copy_visibility(self.vis)
self.vis = apply_gaintable(self.vis, gt, vis_slices=None)
# Now get the control dictionary and calibrate
controls = create_calibration_controls()
controls['T']['first_selfcal'] = 0
calibrated_vis, gaintables = calibrate_function(
self.vis, original, calibration_context='T', controls=controls)
residual = numpy.max(gaintables['T'].residual)
assert residual < 1e-8, "Max T residual = %s" % (residual)
def sum_predict_results(results):
""" Sum a set of predict results of the same shape
:param results: List of visibilities to be summed
:return: summed visibility
"""
sum_results = None
for result in results:
if result is not None:
if sum_results is None:
sum_results = copy_visibility(result)
else:
assert sum_results.data['vis'].shape == result.data['vis'].shape
sum_results.data['vis'] += result.data['vis']
return sum_results
def predict_serial(vis, model: Image, context='2d', vis_slices=1, facets=1, overlap=0, taper=None,
**kwargs) -> Visibility:
"""Predict visibilities using algorithm specified by context
* 2d: Two-dimensional transform
* wstack: wstacking with either vis_slices or wstack (spacing between w planes) set
* wprojection: w projection with wstep (spacing between w places) set, also kernel='wprojection'
* timeslice: snapshot imaging with either vis_slices or timeslice set. timeslice='auto' does every time
* facets: Faceted imaging with facets facets on each axis
* facets_wprojection: facets AND wprojection
* facets_wstack: facets AND wstacking
* wprojection_wstack: wprojection and wstacking
:param vis:
:param model: Model image, used to determine image characteristics
:param context: Imaging context e.g. '2d', 'timeslice', etc.
:param inner: Inner loop 'vis'|'image'
:param kwargs:
:return:
"""
c = imaging_context(context)
vis_iter = c['vis_iterator']
predict = c['predict']
if not isinstance(vis, Visibility):
svis = convert_blockvisibility_to_visibility(vis)
else:
svis = vis
result = copy_visibility(vis, zero=True)
for rows in vis_iter(svis, vis_slices=vis_slices):
if numpy.sum(rows):
visslice = create_visibility_from_rows(svis, rows)
visslice.data['vis'][...] = 0.0
for dpatch in image_scatter_facets(model, facets=facets, overlap=overlap, taper=taper):
result.data['vis'][...] = 0.0
result = predict(visslice, dpatch, **kwargs)
svis.data['vis'][rows] += result.data['vis']
if not isinstance(vis, Visibility):
svis = convert_visibility_to_blockvisibility(svis)
return svis
def rcal_serial(vis: BlockVisibility, components, **kwargs) -> GainTable:
""" Real-time calibration pipeline.
Reads visibilities through a BlockVisibility iterator, calculates model visibilities according to a
component-based sky model, and performs calibration solution, writing a gaintable for each chunk of
visibilities.
:param vis: Visibility or Union(Visibility, Iterable)
:param components: Component-based sky model
:param kwargs: Parameters
:return: gaintable
"""
if not isinstance(vis, collections.Iterable):
vis = [vis]
for ichunk, vischunk in enumerate(vis):
vispred = copy_visibility(vischunk, zero=True)
vispred = predict_skycomponent_visibility(vispred, components)
gt = solve_gaintable(vischunk, vispred, **kwargs)
yield gt
def test_create_vis_iter(self):
vis_iter = create_blockvisibility_iterator(
self.config,
self.times,
self.frequency,
channel_bandwidth=self.channel_bandwidth,
phasecentre=self.phasecentre,
weight=1.0,
polarisation_frame=PolarisationFrame('stokesI'),
integration_time=30.0,
number_integrations=3)
fullvis = None
totalnvis = 0
for i, vis in enumerate(vis_iter):
assert vis.nvis
if i == 0:
fullvis = copy_visibility(vis)
totalnvis = vis.nvis
else:
fullvis = append_visibility(fullvis, vis)
totalnvis += vis.nvis
assert fullvis.nvis == totalnvis
def actualSetup(self,
vnchan=1,
doiso=True,
ntimes=5,
flux_limit=2.0,
zerow=True,
fixed=False):
nfreqwin = vnchan
rmax = 300.0
npixel = 512
cellsize = 0.001
frequency = numpy.linspace(0.8e8, 1.2e8, nfreqwin)
if nfreqwin > 1:
channel_bandwidth = numpy.array(nfreqwin *
[frequency[1] - frequency[0]])
else:
channel_bandwidth = [0.4e8]
times = numpy.linspace(-numpy.pi / 3.0, numpy.pi / 3.0, ntimes)
phasecentre = SkyCoord(ra=-60.0 * u.deg,
dec=-60.0 * u.deg,
frame='icrs',
equinox='J2000')
lowcore = create_named_configuration('LOWBD2', rmax=rmax)
block_vis = create_blockvisibility(
lowcore,
times,
frequency=frequency,
channel_bandwidth=channel_bandwidth,
weight=1.0,
phasecentre=phasecentre,
polarisation_frame=PolarisationFrame("stokesI"),
zerow=zerow)
block_vis.data['uvw'][..., 2] = 0.0
self.beam = create_image_from_visibility(
block_vis,
npixel=npixel,
frequency=[numpy.average(frequency)],
nchan=nfreqwin,
channel_bandwidth=[numpy.sum(channel_bandwidth)],
cellsize=cellsize,
phasecentre=phasecentre)
self.components = create_low_test_skycomponents_from_gleam(
flux_limit=flux_limit,
phasecentre=phasecentre,
frequency=frequency,
polarisation_frame=PolarisationFrame('stokesI'),
radius=npixel * cellsize)
self.beam = create_low_test_beam(self.beam)
self.components = apply_beam_to_skycomponent(self.components,
self.beam,
flux_limit=flux_limit)
self.vis = copy_visibility(block_vis, zero=True)
gt = create_gaintable_from_blockvisibility(block_vis, timeslice='auto')
for i, sc in enumerate(self.components):
if sc.flux[0, 0] > 10:
sc.flux[...] /= 10.0
component_vis = copy_visibility(block_vis, zero=True)
gt = simulate_gaintable(gt,
amplitude_error=0.0,
phase_error=0.1,
seed=None)
component_vis = predict_skycomponent_visibility(component_vis, sc)
component_vis = apply_gaintable(component_vis, gt)
self.vis.data['vis'][...] += component_vis.data['vis'][...]
# Do an isoplanatic selfcal
self.model_vis = copy_visibility(self.vis, zero=True)
self.model_vis = predict_skycomponent_visibility(
self.model_vis, self.components)
if doiso:
gt = solve_gaintable(self.vis,
self.model_vis,
phase_only=True,
timeslice='auto')
self.vis = apply_gaintable(self.vis, gt, inverse=True)
self.model_vis = convert_blockvisibility_to_visibility(self.model_vis)
self.model_vis, _, _ = weight_visibility(self.model_vis, self.beam)
self.dirty_model, sumwt = invert_function(self.model_vis,
self.beam,
context='2d')
export_image_to_fits(self.dirty_model,
"%s/test_skymodel-model_dirty.fits" % self.dir)
lvis = convert_blockvisibility_to_visibility(self.vis)
lvis, _, _ = weight_visibility(lvis, self.beam)
dirty, sumwt = invert_function(lvis, self.beam, context='2d')
if doiso:
export_image_to_fits(
dirty, "%s/test_skymodel-initial-iso-residual.fits" % self.dir)
else:
export_image_to_fits(
dirty,
"%s/test_skymodel-initial-noiso-residual.fits" % self.dir)
self.skymodels = [
SkyModel(components=[cm], fixed=fixed) for cm in self.components
]
def ical_serial(block_vis: BlockVisibility,
model: Image,
components=None,
context='2d',
controls=None,
**kwargs):
""" Post observation image, deconvolve, and self-calibrate
:param vis:
:param model: Model image
:param components: Initial components
:param context: Imaging context
:param controls: calibration controls dictionary
:return: model, residual, restored
"""
nmajor = get_parameter(kwargs, 'nmajor', 5)
log.info("ical_serial: Performing %d major cycles" % nmajor)
do_selfcal = get_parameter(kwargs, "do_selfcal", False)
if controls is None:
controls = create_calibration_controls(**kwargs)
# The model is added to each major cycle and then the visibilities are
# calculated from the full model
vis = convert_blockvisibility_to_visibility(block_vis)
block_vispred = copy_visibility(block_vis, zero=True)
vispred = convert_blockvisibility_to_visibility(block_vispred)
vispred.data['vis'][...] = 0.0
visres = copy_visibility(vispred)
vispred = predict_serial(vispred, model, context=context, **kwargs)
if components is not None:
vispred = predict_skycomponent_visibility(vispred, components)
if do_selfcal:
vis, gaintables = calibrate_function(vis,
vispred,
'TGB',
controls,
iteration=-1)
visres.data['vis'] = vis.data['vis'] - vispred.data['vis']
dirty, sumwt = invert_serial(visres, model, context=context, **kwargs)
log.info("Maximum in residual image is %.6f" %
(numpy.max(numpy.abs(dirty.data))))
psf, sumwt = invert_serial(visres,
model,
dopsf=True,
context=context,
**kwargs)
thresh = get_parameter(kwargs, "threshold", 0.0)
for i in range(nmajor):
log.info("ical_serial: Start of major cycle %d of %d" % (i, nmajor))
cc, res = deconvolve_cube(dirty, psf, **kwargs)
model.data += cc.data
vispred.data['vis'][...] = 0.0
vispred = predict_serial(vispred, model, context=context, **kwargs)
if do_selfcal:
vis, gaintables = calibrate_function(vis,
vispred,
'TGB',
controls,
iteration=i)
visres.data['vis'] = vis.data['vis'] - vispred.data['vis']
dirty, sumwt = invert_serial(visres, model, context=context, **kwargs)
log.info("Maximum in residual image is %s" %
(numpy.max(numpy.abs(dirty.data))))
if numpy.abs(dirty.data).max() < 1.1 * thresh:
log.info("ical_serial: Reached stopping threshold %.6f Jy" %
thresh)
break
log.info("ical_serial: End of major cycle")
log.info("ical_serial: End of major cycles")
restored = restore_cube(model, psf, dirty, **kwargs)
return model, dirty, restored