def core_solve(self,
spf,
dpf,
phase_error=0.1,
amplitude_error=0.0,
leakage=0.0,
phase_only=True,
niter=200,
crosspol=False,
residual_tol=1e-6,
f=None,
vnchan=3):
if f is None:
f = [100.0, 50.0, -10.0, 40.0]
self.actualSetup(spf, dpf, f=f, vnchan=vnchan)
gt = create_gaintable_from_blockvisibility(self.vis)
log.info("Created gain table: %s" % (gaintable_summary(gt)))
gt = simulate_gaintable(gt,
phase_error=phase_error,
amplitude_error=amplitude_error,
leakage=leakage)
original = copy_visibility(self.vis)
vis = apply_gaintable(self.vis, gt)
gtsol = solve_gaintable(self.vis,
original,
phase_only=phase_only,
niter=niter,
crosspol=crosspol,
tol=1e-6)
vis = apply_gaintable(vis, gtsol, inverse=True)
residual = numpy.max(gtsol.residual)
assert residual < residual_tol, "%s %s Max residual = %s" % (spf, dpf,
residual)
log.debug(qa_gaintable(gt))
assert numpy.max(numpy.abs(gtsol.gain - 1.0)) > 0.1
def test_apply_gaintable_and_inverse_both(self):
for spf, dpf in[('stokesI', 'stokesI'), ('stokesIQUV', 'linear'), ('stokesIQUV', 'circular')]:
self.actualSetup(spf, dpf)
gt = create_gaintable_from_blockvisibility(self.vis, timeslice='auto')
log.info("Created gain table: %s" % (gaintable_summary(gt)))
gt = simulate_gaintable(gt, phase_error=0.1, amplitude_error=0.1)
original = copy_visibility(self.vis)
vis = apply_gaintable(self.vis, gt)
vis = apply_gaintable(self.vis, gt, inverse=True)
error = numpy.max(numpy.abs(vis.vis - original.vis))
assert error < 1e-12, "Error = %s" % (error)
def insert_unittest_errors(vt, seed=180555, amp_errors=None, phase_errors=None):
"""Simulate gain errors and apply
:param vt:
:param seed: Random number seed, set to big integer repeat values from run to run
:param phase_errors: e.g. {'T': 1.0, 'G': 0.1, 'B': 0.01}
:param amp_errors: e.g. {'T': 0.0, 'G': 0.01, 'B': 0.01}
:return:
"""
numpy.random.seed(seed)
controls = create_calibration_controls()
if amp_errors is None:
amp_errors = {'T': 0.0, 'G': 0.01, 'B': 0.01}
if phase_errors is None:
phase_errors = {'T': 1.0, 'G': 0.1, 'B': 0.01}
for c in "TGB":
gaintable = \
create_gaintable_from_blockvisibility(vt, timeslice=controls[c]['timeslice'])
gaintable = simulate_gaintable(gaintable,
timeslice=controls[c]['timeslice'],
phase_only=controls[c]['phase_only'],
crosspol=controls[c]['shape'] == 'matrix',
phase_error=phase_errors[c], amplitude_error=amp_errors[c])
vt = apply_gaintable(vt, gaintable, inverse=True, timeslice=controls[c]['timeslice'])
return vt
def test_apply_gaintable_null(self):
for spf, dpf in[('stokesI', 'stokesI'), ('stokesIQUV', 'linear'), ('stokesIQUV', 'circular')]:
self.actualSetup(spf, dpf)
gt = create_gaintable_from_blockvisibility(self.vis, timeslice='auto')
gt.data['gain']*=0.0
original = copy_visibility(self.vis)
vis = apply_gaintable(self.vis, gt, inverse=True)
error = numpy.max(numpy.abs(vis.vis[:,0,1,...] - original.vis[:,0,1,...]))
assert error < 1e-12, "Error = %s" % (error)
def create_blockvisibility_iterator(config: Configuration, times: numpy.array, frequency: numpy.array,
channel_bandwidth, phasecentre: SkyCoord, weight: float = 1,
polarisation_frame=PolarisationFrame('stokesI'), integration_time=1.0,
number_integrations=1, predict=predict_2d, model=None, components=None,
phase_error=0.0, amplitude_error=0.0, sleep=0.0, **kwargs):
""" Create a sequence of Visibilities and optionally predicting and coalescing
This is useful mainly for performing large simulations. Do something like::
vis_iter = create_blockvisibility_iterator(config, times, frequency, channel_bandwidth, phasecentre=phasecentre,
weight=1.0, integration_time=30.0, number_integrations=3)
for i, vis in enumerate(vis_iter):
if i == 0:
fullvis = vis
else:
fullvis = append_visibility(fullvis, vis)
:param config: Configuration of antennas
:param times: hour angles in radians
:param frequency: frequencies (Hz] Shape [nchan]
:param weight: weight of a single sample
:param phasecentre: phasecentre of observation
:param npol: Number of polarizations
:param integration_time: Integration time ('auto' or value in s)
:param number_integrations: Number of integrations to be created at each time.
:param model: Model image to be inserted
:param components: Components to be inserted
:param sleep_time: Time to sleep between yields
:return: Visibility
"""
for time in times:
actualtimes = time + numpy.arange(0, number_integrations) * integration_time * numpy.pi / 43200.0
bvis = create_blockvisibility(config, actualtimes, frequency=frequency, phasecentre=phasecentre, weight=weight,
polarisation_frame=polarisation_frame, integration_time=integration_time,
channel_bandwidth=channel_bandwidth)
if model is not None:
vis = predict(bvis, model, **kwargs)
bvis = convert_visibility_to_blockvisibility(vis)
if components is not None:
bvis = predict_skycomponent_visibility(bvis, components)
# Add phase errors
if phase_error > 0.0 or amplitude_error > 0.0:
gt = create_gaintable_from_blockvisibility(bvis)
gt = simulate_gaintable(gt=gt, phase_error=phase_error, amplitude_error=amplitude_error)
bvis = apply_gaintable(bvis, gt)
import time
time.sleep(sleep)
yield bvis
def make_e(vis, calskymodel, evis_all):
# Return the estep for a given skymodel
evis = copy_visibility(vis)
tvis = copy_visibility(vis, zero=True)
tvis = predict_skymodel_visibility(tvis, calskymodel[0])
tvis = apply_gaintable(tvis, calskymodel[1])
# E step is the data model for a window plus the difference between the observed data_models
# and the summed data models or, put another way, its the observed data minus the
# summed visibility for all other windows
evis.data['vis'][...] = tvis.data['vis'][...] + vis.data['vis'][...] - evis_all.data['vis'][...]
return evis
def test_create_gaintable_from_visibility(self):
for spf, dpf in[('stokesI', 'stokesI'), ('stokesIQUV', 'linear'), ('stokesIQUV', 'circular')]:
self.actualSetup(spf, dpf)
gt = create_gaintable_from_blockvisibility(self.vis, timeslice='auto')
log.info("Created gain table: %s" % (gaintable_summary(gt)))
gt = simulate_gaintable(gt, phase_error=1.0)
original = copy_visibility(self.vis)
vis = apply_gaintable(self.vis, gt)
assert numpy.max(numpy.abs(original.vis)) > 0.0
assert numpy.max(numpy.abs(vis.vis)) > 0.0
assert numpy.max(numpy.abs(vis.vis - original.vis)) > 0.0
def test_solve_gaintable_scalar_bandpass(self):
self.actualSetup('stokesI', 'stokesI', f=[100.0], vnchan=128)
gt = create_gaintable_from_blockvisibility(self.vis)
log.info("Created gain table: %s" % (gaintable_summary(gt)))
gt = simulate_gaintable(gt, phase_error=10.0, amplitude_error=0.01, smooth_channels=8)
original = copy_visibility(self.vis)
self.vis = apply_gaintable(self.vis, gt)
gtsol = solve_gaintable(self.vis, original, phase_only=False, niter=200)
residual = numpy.max(gtsol.residual)
assert residual < 3e-8, "Max residual = %s" % (residual)
assert numpy.max(numpy.abs(gtsol.gain - 1.0)) > 0.1
def test_solve_gaintable_scalar_timeslice(self):
self.actualSetup('stokesI', 'stokesI', f=[100.0], ntimes=10)
gt = create_gaintable_from_blockvisibility(self.vis, timeslice=120.0)
log.info("Created gain table: %s" % (gaintable_summary(gt)))
gt = simulate_gaintable(gt, phase_error=10.0, amplitude_error=0.0)
original = copy_visibility(self.vis)
self.vis = apply_gaintable(self.vis, gt)
gtsol = solve_gaintable(self.vis, original, phase_only=True, niter=200)
residual = numpy.max(gtsol.residual)
assert residual < 3e-8, "Max residual = %s" % (residual)
assert numpy.max(numpy.abs(gtsol.gain - 1.0)) > 0.1
def test_solve_gaintable_scalar_normalise(self):
self.actualSetup('stokesI', 'stokesI', f=[100.0])
gt = create_gaintable_from_blockvisibility(self.vis)
log.info("Created gain table: %s" % (gaintable_summary(gt)))
gt = simulate_gaintable(gt, phase_error=0.0, amplitude_error=0.1)
gt.data['gain'] *= 2.0
original = copy_visibility(self.vis)
self.vis = apply_gaintable(self.vis, gt)
gtsol = solve_gaintable(self.vis, original, phase_only=False, niter=200, normalise_gains=True)
residual = numpy.max(gtsol.residual)
assert residual < 3e-8, "Max residual = %s" % (residual)
assert numpy.max(numpy.abs(gtsol.gain - 1.0)) > 0.1
def modelpartition_list_expectation_step(vis: BlockVisibility, evis_all: BlockVisibility, modelpartition, **kwargs):
"""Calculates E step in equation A12
This is the data model for this window plus the difference between observed data and summed data models
:param evis_all: Sum data models
:param csm: csm element being fit
:param kwargs:
:return: Data model (i.e. visibility) for this csm
"""
evis = copy_visibility(evis_all)
tvis = copy_visibility(vis, zero=True)
tvis = predict_skymodel_visibility(tvis, modelpartition[0], **kwargs)
tvis = apply_gaintable(tvis, modelpartition[1])
evis.data['vis'][...] = tvis.data['vis'][...] + vis.data['vis'][...] - evis_all.data['vis'][...]
return evis
def calskymodel_expectation_all(vis: BlockVisibility, calskymodels, **kwargs):
"""Calculates E step in equation A12
This is the sum of the data models over all skymodel
:param vis: Visibility
:param csm: List of (skymodel, gaintable) tuples
:param kwargs:
:return: Sum of data models (i.e. a visibility)
"""
evis = copy_visibility(vis, zero=True)
tvis = copy_visibility(vis, zero=True)
for csm in calskymodels:
tvis.data['vis'][...] = 0.0
tvis = predict_skymodel_visibility(tvis, csm[0], **kwargs)
tvis = apply_gaintable(tvis, csm[1])
evis.data['vis'][...] += tvis.data['vis'][...]
return evis
def calibrate_list_serial_workflow(vis_list,
model_vislist,
calibration_context='TG',
global_solution=True,
**kwargs):
""" Create a set of components for (optionally global) calibration of a list of visibilities
If global solution is true then visibilities are gathered to a single visibility data set which is then
self-calibrated. The resulting gaintable is then effectively scattered out for application to each visibility
set. If global solution is false then the solutions are performed locally.
:param vis_list:
:param model_vislist:
:param calibration_context: String giving terms to be calibrated e.g. 'TGB'
:param global_solution: Solve for global gains
:param kwargs: Parameters for functions in components
:return:
"""
def solve_and_apply(vis, modelvis=None):
return calibrate_function(vis,
modelvis,
calibration_context=calibration_context,
**kwargs)[0]
if global_solution:
point_vislist = [
divide_visibility(vis_list[i], model_vislist[i])
for i, _ in enumerate(vis_list)
]
global_point_vis_list = visibility_gather_channel(point_vislist)
global_point_vis_list = integrate_visibility_by_channel(
global_point_vis_list)
# This is a global solution so we only compute one gain table
_, gt_list = solve_and_apply(global_point_vis_list)
return [apply_gaintable(v, gt_list, inverse=True) for v in vis_list]
else:
return [
solve_and_apply(vis_list[i], model_vislist[i])
for i, v in enumerate(vis_list)
]
def ingest_visibility(self,
freq=None,
chan_width=None,
times=None,
add_errors=False,
block=True,
bandpass=False):
if freq is None:
freq = [1e8]
if chan_width is None:
chan_width = [1e6]
if times is None:
times = (numpy.pi / 12.0) * numpy.linspace(-3.0, 3.0, 5)
lowcore = create_named_configuration('LOWBD2', rmax=750.0)
frequency = numpy.array(freq)
channel_bandwidth = numpy.array(chan_width)
phasecentre = SkyCoord(ra=+180.0 * u.deg,
dec=-60.0 * u.deg,
frame='icrs',
equinox='J2000')
if block:
vt = create_blockvisibility(
lowcore,
times,
frequency,
channel_bandwidth=channel_bandwidth,
weight=1.0,
phasecentre=phasecentre,
polarisation_frame=PolarisationFrame("stokesI"))
else:
vt = create_visibility(
lowcore,
times,
frequency,
channel_bandwidth=channel_bandwidth,
weight=1.0,
phasecentre=phasecentre,
polarisation_frame=PolarisationFrame("stokesI"))
cellsize = 0.001
model = create_image_from_visibility(
vt,
npixel=self.npixel,
cellsize=cellsize,
npol=1,
frequency=frequency,
phasecentre=phasecentre,
polarisation_frame=PolarisationFrame("stokesI"))
nchan = len(self.frequency)
flux = numpy.array(nchan * [[100.0]])
facets = 4
rpix = model.wcs.wcs.crpix - 1.0
spacing_pixels = self.npixel // facets
centers = [-1.5, -0.5, 0.5, 1.5]
comps = list()
for iy in centers:
for ix in centers:
p = int(round(rpix[0] + ix * spacing_pixels * numpy.sign(model.wcs.wcs.cdelt[0]))), \
int(round(rpix[1] + iy * spacing_pixels * numpy.sign(model.wcs.wcs.cdelt[1])))
sc = pixel_to_skycoord(p[0], p[1], model.wcs, origin=1)
comp = create_skycomponent(
direction=sc,
flux=flux,
frequency=frequency,
polarisation_frame=PolarisationFrame("stokesI"))
comps.append(comp)
if block:
predict_skycomponent_visibility(vt, comps)
else:
predict_skycomponent_visibility(vt, comps)
insert_skycomponent(model, comps)
self.comps = comps
self.model = copy_image(model)
self.empty_model = create_empty_image_like(model)
export_image_to_fits(
model, '%s/test_pipeline_functions_model.fits' % (self.dir))
if add_errors:
# These will be the same for all calls
numpy.random.seed(180555)
gt = create_gaintable_from_blockvisibility(vt)
gt = simulate_gaintable(gt, phase_error=1.0, amplitude_error=0.0)
vt = apply_gaintable(vt, gt)
if bandpass:
bgt = create_gaintable_from_blockvisibility(vt, timeslice=1e5)
bgt = simulate_gaintable(bgt,
phase_error=0.01,
amplitude_error=0.01,
smooth_channels=4)
vt = apply_gaintable(vt, bgt)
return vt
def actualSetup(self,
vnchan=1,
doiso=True,
ntimes=5,
flux_limit=2.0,
zerow=True,
fixed=False):
nfreqwin = vnchan
rmax = 300.0
npixel = 512
cellsize = 0.001
frequency = numpy.linspace(0.8e8, 1.2e8, nfreqwin)
if nfreqwin > 1:
channel_bandwidth = numpy.array(nfreqwin *
[frequency[1] - frequency[0]])
else:
channel_bandwidth = [0.4e8]
times = numpy.linspace(-numpy.pi / 3.0, numpy.pi / 3.0, ntimes)
phasecentre = SkyCoord(ra=-60.0 * u.deg,
dec=-60.0 * u.deg,
frame='icrs',
equinox='J2000')
lowcore = create_named_configuration('LOWBD2', rmax=rmax)
block_vis = create_blockvisibility(
lowcore,
times,
frequency=frequency,
channel_bandwidth=channel_bandwidth,
weight=1.0,
phasecentre=phasecentre,
polarisation_frame=PolarisationFrame("stokesI"),
zerow=zerow)
block_vis.data['uvw'][..., 2] = 0.0
self.beam = create_image_from_visibility(
block_vis,
npixel=npixel,
frequency=[numpy.average(frequency)],
nchan=nfreqwin,
channel_bandwidth=[numpy.sum(channel_bandwidth)],
cellsize=cellsize,
phasecentre=phasecentre)
self.components = create_low_test_skycomponents_from_gleam(
flux_limit=flux_limit,
phasecentre=phasecentre,
frequency=frequency,
polarisation_frame=PolarisationFrame('stokesI'),
radius=npixel * cellsize)
self.beam = create_low_test_beam(self.beam)
self.components = apply_beam_to_skycomponent(self.components,
self.beam,
flux_limit=flux_limit)
self.vis = copy_visibility(block_vis, zero=True)
gt = create_gaintable_from_blockvisibility(block_vis, timeslice='auto')
for i, sc in enumerate(self.components):
if sc.flux[0, 0] > 10:
sc.flux[...] /= 10.0
component_vis = copy_visibility(block_vis, zero=True)
gt = simulate_gaintable(gt,
amplitude_error=0.0,
phase_error=0.1,
seed=None)
component_vis = predict_skycomponent_visibility(component_vis, sc)
component_vis = apply_gaintable(component_vis, gt)
self.vis.data['vis'][...] += component_vis.data['vis'][...]
# Do an isoplanatic selfcal
self.model_vis = copy_visibility(self.vis, zero=True)
self.model_vis = predict_skycomponent_visibility(
self.model_vis, self.components)
if doiso:
gt = solve_gaintable(self.vis,
self.model_vis,
phase_only=True,
timeslice='auto')
self.vis = apply_gaintable(self.vis, gt, inverse=True)
self.model_vis = convert_blockvisibility_to_visibility(self.model_vis)
self.model_vis, _, _ = weight_visibility(self.model_vis, self.beam)
self.dirty_model, sumwt = invert_function(self.model_vis,
self.beam,
context='2d')
export_image_to_fits(self.dirty_model,
"%s/test_skymodel-model_dirty.fits" % self.dir)
lvis = convert_blockvisibility_to_visibility(self.vis)
lvis, _, _ = weight_visibility(lvis, self.beam)
dirty, sumwt = invert_function(lvis, self.beam, context='2d')
if doiso:
export_image_to_fits(
dirty, "%s/test_skymodel-initial-iso-residual.fits" % self.dir)
else:
export_image_to_fits(
dirty,
"%s/test_skymodel-initial-noiso-residual.fits" % self.dir)
self.skymodels = [
SkyModel(components=[cm], fixed=fixed) for cm in self.components
]
block_vis = convert_visibility_to_blockvisibility(predicted_vis)
#print("np.sum(block_vis.data): ", numpy.sum(block_vis.data['vis']))
#print("nchan npol nants ", block_vis.nchan, block_vis.npol, block_vis.nants)
#print("uvw", block_vis.uvw, numpy.sum(block_vis.uvw))
#print("vis", block_vis.vis, numpy.sum(block_vis.vis))
#print("weight", block_vis.weight, numpy.sum(block_vis.weight))
#print("time", block_vis.time, numpy.sum(block_vis.time))
#print("integration_time", block_vis.integration_time, numpy.sum(block_vis.integration_time))
#print("nvis, size", block_vis.nvis, block_vis.size())
gt = create_gaintable_from_blockvisibility(block_vis)
#print("np.sum(gt.data): ", numpy.sum(gt.data['gain']))
gt = simulate_gaintable(gt, phase_error=1.0)
#print("np.sum(gt.data): ", numpy.sum(gt.data['gain']))
blockvis = apply_gaintable(block_vis, gt)
#print("np.sum(blockvis.data): ", numpy.sum(blockvis.data['vis']))
model = create_image_from_visibility(
block_vis,
npixel=npixel,
frequency=[numpy.average(frequency)],
nchan=1,
channel_bandwidth=[numpy.sum(channel_bandwidth)],
cellsize=cellsize,
phasecentre=phasecentre)
#print("model sum, min, max, shape: ", numpy.sum(model.data), numpy.amin(model.data), numpy.amax(model.data), model.shape)
print(qa_image(model, context='Blockvis model image'))
export_image_to_fits(model, '%s/imaging-blockvis_model.fits' % (results_dir))
def predict_and_apply(ovis, calskymodel):
tvis = copy_visibility(ovis, zero=True)
tvis = predict_skymodel_visibility(tvis, calskymodel[0])
tvis = apply_gaintable(tvis, calskymodel[1])
return tvis
def corrupt_vis(vis, gt, **kwargs):
if gt is None:
gt = create_gaintable_from_blockvisibility(vis, **kwargs)
gt = simulate_gaintable(gt, **kwargs)
return apply_gaintable(vis, gt)
def predict_and_apply(ovis, modelpartition):
tvis = copy_visibility(ovis, zero=True)
tvis = predict_skymodel_visibility(tvis, modelpartition[0])
tvis = apply_gaintable(tvis, modelpartition[1])
return tvis
