def test_apply_gaintable_and_inverse_both(self):
for spf, dpf in[('stokesI', 'stokesI'), ('stokesIQUV', 'linear'), ('stokesIQUV', 'circular')]:
self.actualSetup(spf, dpf)
gt = create_gaintable_from_blockvisibility(self.vis, timeslice='auto')
log.info("Created gain table: %s" % (gaintable_summary(gt)))
gt = simulate_gaintable(gt, phase_error=0.1, amplitude_error=0.1)
original = copy_visibility(self.vis)
vis = apply_gaintable(self.vis, gt)
vis = apply_gaintable(self.vis, gt, inverse=True)
error = numpy.max(numpy.abs(vis.vis - original.vis))
assert error < 1e-12, "Error = %s" % (error)
def test_solve_gaintable_stokesI_phaseonly(self):
self.actualSetup('stokesI', 'stokesI', f=[100.0])
gt = create_gaintable_from_blockvisibility(self.vis)
log.info("Created gain table: %s" % (gaintable_summary(gt)))
gt = simulate_gaintable(gt, phase_error=10.0, amplitude_error=0.0)
original = copy_visibility(self.vis)
self.vis = apply_gaintable(self.vis, gt)
gtsol = solve_gaintable(self.vis, original, phase_only=True, niter=200)
self.vis = apply_gaintable(self.vis, gtsol)
residual = numpy.max(gtsol.residual)
assert residual < 3e-8, "Max residual = %s" % (residual)
assert numpy.max(numpy.abs(gtsol.gain - 1.0)) > 0.1
def test_solve_gaintable_stokesI_small_n_large_t(self):
# Select only 6 stations
self.actualSetup('stokesI', 'stokesI', f=[100.0], ntimes=4000, rmax=83)
gt = create_gaintable_from_blockvisibility(self.vis)
log.info("Created gain table: %s" % (gaintable_summary(gt)))
gt = simulate_gaintable(gt, phase_error=10.0, amplitude_error=0.0)
gt.data['gain'] = gt.gain[1, ...]
original = copy_visibility(self.vis)
self.vis = apply_gaintable(self.vis, gt)
gtsol = solve_gaintable(self.vis, original, phase_only=True, niter=200)
self.vis = apply_gaintable(self.vis, gtsol)
residual = numpy.max(gtsol.residual)
assert residual < 3e-8, "Max residual = %s" % (residual)
assert numpy.max(numpy.abs(gtsol.gain - 1.0)) > 0.1
def test_apply_gaintable_null(self):
for spf, dpf in[('stokesI', 'stokesI'), ('stokesIQUV', 'linear'), ('stokesIQUV', 'circular')]:
self.actualSetup(spf, dpf)
gt = create_gaintable_from_blockvisibility(self.vis, timeslice='auto')
gt.data['gain']*=0.0
original = copy_visibility(self.vis)
vis = apply_gaintable(self.vis, gt, inverse=True)
error = numpy.max(numpy.abs(vis.vis[:,0,1,...] - original.vis[:,0,1,...]))
assert error < 1e-12, "Error = %s" % (error)
def solve_calibrate_chain(vis, model_vis, calibration_context='T', controls=None, iteration=0, tol=1e-6, **kwargs):
""" Calibrate using algorithm specified by calibration_context
The context string can denote a sequence of calibrations e.g. TGB with different timescales.
:param vis:
:param model_vis:
:param calibration_context: calibration contexts in order of correction e.g. 'TGB'
:param controls: controls dictionary, modified as necessary
:param iteration: Iteration number to be compared to the 'first_selfcal' field.
:param kwargs:
:return: Calibrated data_models, dict(gaintables)
"""
gaintables = {}
if controls is None:
controls = create_calibration_controls()
isVis = isinstance(vis, Visibility)
if isVis:
avis = convert_visibility_to_blockvisibility(vis)
else:
avis = vis
isMVis = isinstance(model_vis, Visibility)
if isMVis:
amvis = convert_visibility_to_blockvisibility(model_vis)
else:
amvis = model_vis
assert isinstance(avis, BlockVisibility), avis
assert amvis.__repr__() != avis.__repr__(), "Vis and model vis are the same object: convert problem"
# Always return a gain table, even if null
for c in calibration_context:
gaintables[c] = \
create_gaintable_from_blockvisibility(avis, timeslice=controls[c]['timeslice'])
if iteration >= controls[c]['first_selfcal']:
if numpy.max(numpy.abs(vis.weight)) > 0.0 and (amvis is None or numpy.max(numpy.abs(amvis.vis)) > 0.0):
gaintables[c] = solve_gaintable(avis, amvis,
timeslice=controls[c]['timeslice'],
phase_only=controls[c]['phase_only'],
crosspol=controls[c]['shape'] == 'matrix',
tol=tol)
log.debug('calibrate_chain: Jones matrix %s, iteration %d' % (c, iteration))
log.debug(qa_gaintable(gaintables[c], context='Jones matrix %s, iteration %d' % (c, iteration)))
avis = apply_gaintable(avis, gaintables[c], inverse=True, timeslice=controls[c]['timeslice'])
else:
log.debug('calibrate_chain: Jones matrix %s not solved, iteration %d' % (c, iteration))
else:
log.debug('calibrate_chain: Jones matrix %s not solved, iteration %d' % (c, iteration))
return gaintables
def test_create_gaintable_from_visibility(self):
for spf, dpf in[('stokesI', 'stokesI'), ('stokesIQUV', 'linear'), ('stokesIQUV', 'circular')]:
self.actualSetup(spf, dpf)
gt = create_gaintable_from_blockvisibility(self.vis, timeslice='auto')
log.info("Created gain table: %s" % (gaintable_summary(gt)))
gt = simulate_gaintable(gt, phase_error=1.0)
original = copy_visibility(self.vis)
vis = apply_gaintable(self.vis, gt)
assert numpy.max(numpy.abs(original.vis)) > 0.0
assert numpy.max(numpy.abs(vis.vis)) > 0.0
assert numpy.max(numpy.abs(vis.vis - original.vis)) > 0.0
def apply_calibration_chain(vis,
gaintables,
calibration_context='T',
controls=None,
iteration=0,
tol=1e-6,
**kwargs):
""" Calibrate using algorithm specified by calibration_context and the calibration controls
The context string can denote a sequence of calibrations e.g. TGB with different timescales.
:param vis:
:param model_vis:
:param calibration_context: calibration contexts in order of correction e.g. 'TGB'
:param control: controls dictionary, modified as necessary
:param iteration: Iteration number to be compared to the 'first_selfcal' field.
:param kwargs:
:return: Calibrated data_models, dict(gaintables)
"""
if controls is None:
controls = create_calibration_controls()
# Check to see if changes are required
changes = False
for c in calibration_context:
if (iteration >=
controls[c]['first_selfcal']) and (c in gaintables.keys()):
changes = True
if changes:
isVis = isinstance(vis, Visibility)
if isVis:
avis = convert_visibility_to_blockvisibility(vis)
else:
avis = vis
assert isinstance(avis, BlockVisibility), avis
for c in calibration_context:
if iteration >= controls[c]['first_selfcal']:
avis = apply_gaintable(avis,
gaintables[c],
timeslice=controls[c]['timeslice'])
if isVis:
return convert_blockvisibility_to_visibility(avis)
else:
return avis
else:
return vis
def core_solve(self,
spf,
dpf,
phase_error=0.1,
amplitude_error=0.0,
leakage=0.0,
phase_only=True,
niter=200,
crosspol=False,
residual_tol=1e-6,
f=None,
vnchan=3,
timeslice='auto'):
if f is None:
f = [100.0, 50.0, -10.0, 40.0]
self.actualSetup(spf, dpf, f=f, vnchan=vnchan)
gt = create_gaintable_from_blockvisibility(self.vis,
timeslice=timeslice)
log.info("Created gain table: %s" % (gaintable_summary(gt)))
gt = simulate_gaintable(gt,
phase_error=phase_error,
amplitude_error=amplitude_error,
leakage=leakage)
original = copy_visibility(self.vis)
vis = apply_gaintable(self.vis, gt)
gtsol = solve_gaintable(self.vis,
original,
phase_only=phase_only,
niter=niter,
crosspol=crosspol,
tol=1e-6)
vis = apply_gaintable(vis, gtsol, inverse=True)
residual = numpy.max(gtsol.residual)
assert residual < residual_tol, "%s %s Max residual = %s" % (spf, dpf,
residual)
log.debug(qa_gaintable(gt))
assert numpy.max(numpy.abs(gtsol.gain - 1.0)) > 0.1
def test_solve_gaintable_stokesI_bandpass(self):
self.actualSetup('stokesI', 'stokesI', f=[100.0], vnchan=128)
gt = create_gaintable_from_blockvisibility(self.vis)
log.info("Created gain table: %s" % (gaintable_summary(gt)))
gt = simulate_gaintable(gt,
phase_error=10.0,
amplitude_error=0.01,
smooth_channels=8)
original = copy_visibility(self.vis)
self.vis = apply_gaintable(self.vis, gt)
gtsol = solve_gaintable(self.vis,
original,
phase_only=False,
niter=200,
damping=0.5)
residual = numpy.max(gtsol.residual)
assert residual < 3e-8, "Max residual = %s" % (residual)
assert numpy.max(numpy.abs(gtsol.gain - 1.0)) > 0.1
def insert_unittest_errors(vt,
seed=180555,
calibration_context="TG",
amp_errors=None,
phase_errors=None):
"""Simulate gain errors and apply
:param vt:
:param seed: Random number seed, set to big integer repeat values from run to run
:param phase_errors: e.g. {'T': 1.0, 'G': 0.1, 'B': 0.01}
:param amp_errors: e.g. {'T': 0.0, 'G': 0.01, 'B': 0.01}
:return:
"""
controls = create_calibration_controls()
if amp_errors is None:
amp_errors = {'T': 0.0, 'G': 0.01, 'B': 0.01}
if phase_errors is None:
phase_errors = {'T': 1.0, 'G': 0.1, 'B': 0.01}
for c in calibration_context:
gaintable = create_gaintable_from_blockvisibility(
vt, timeslice=controls[c]['timeslice'])
gaintable = simulate_gaintable(
gaintable,
phase_error=phase_errors[c],
amplitude_error=amp_errors[c],
timeslice=controls[c]['timeslice'],
phase_only=controls[c]['phase_only'],
crosspol=controls[c]['shape'] == 'matrix')
vt = apply_gaintable(vt,
gaintable,
inverse=True,
timeslice=controls[c]['timeslice'])
return vt
def calibrate_chain(vis,
model_vis,
gaintables=None,
calibration_context='T',
controls=None,
iteration=0,
tol=1e-8,
**kwargs):
""" Calibrate using algorithm specified by calibration_context
The context string can denote a sequence of calibrations e.g. TGB with different timescales.
:param vis:
:param model_vis:
:param calibration_context: calibration contexts in order of correction e.g. 'TGB'
:param controls: controls dictionary, modified as necessary
:param iteration: Iteration number to be compared to the 'first_selfcal' field.
:param kwargs:
:return: Calibrated data_models, dict(gaintables)
"""
if controls is None:
controls = create_calibration_controls()
# Check to see if changes are required
changes = False
for c in calibration_context:
if iteration >= controls[c]['first_selfcal']:
changes = True
if changes:
isVis = isinstance(vis, Visibility)
if isVis:
avis = convert_visibility_to_blockvisibility(vis)
else:
avis = vis
isMVis = isinstance(model_vis, Visibility)
if isMVis:
amvis = convert_visibility_to_blockvisibility(model_vis)
else:
amvis = model_vis
assert isinstance(avis, BlockVisibility), avis
if gaintables is None:
gaintables = dict()
for c in calibration_context:
if iteration >= controls[c]['first_selfcal']:
if c not in gaintables.keys():
log.info("Creating new {} gaintable".format(c))
gaintables[c] = \
create_gaintable_from_blockvisibility(avis,
timeslice=controls[c]['timeslice'])
gaintables[c] = solve_gaintable(
avis,
amvis,
gt=gaintables[c],
timeslice=controls[c]['timeslice'],
phase_only=controls[c]['phase_only'],
crosspol=controls[c]['shape'] == 'matrix',
tol=tol)
log.debug('calibrate_chain: Jones matrix %s, iteration %d' %
(c, iteration))
log.debug(
qa_gaintable(gaintables[c],
context='Jones matrix %s, iteration %d' %
(c, iteration)))
avis = apply_gaintable(avis,
gaintables[c],
inverse=True,
timeslice=controls[c]['timeslice'])
else:
log.debug(
'calibrate_chain: Jones matrix %s not solved, iteration %d'
% (c, iteration))
if isVis:
return convert_blockvisibility_to_visibility(avis), gaintables
else:
return avis, gaintables
else:
return vis, gaintables
def actualSetUp(self,
nfreqwin=3,
dospectral=True,
dopol=False,
amp_errors=None,
phase_errors=None,
zerow=True):
if amp_errors is None:
amp_errors = {'T': 0.0, 'G': 0.1}
if phase_errors is None:
phase_errors = {'T': 1.0, 'G': 0.0}
self.npixel = 512
self.low = create_named_configuration('LOWBD2', rmax=750.0)
self.freqwin = nfreqwin
self.vis_list = list()
self.ntimes = 1
self.times = numpy.linspace(-3.0, +3.0, self.ntimes) * numpy.pi / 12.0
self.frequency = numpy.linspace(0.8e8, 1.2e8, self.freqwin)
if self.freqwin > 1:
self.channelwidth = numpy.array(
self.freqwin * [self.frequency[1] - self.frequency[0]])
else:
self.channelwidth = numpy.array([1e6])
if dopol:
self.vis_pol = PolarisationFrame('linear')
self.image_pol = PolarisationFrame('stokesIQUV')
f = numpy.array([100.0, 20.0, -10.0, 1.0])
else:
self.vis_pol = PolarisationFrame('stokesI')
self.image_pol = PolarisationFrame('stokesI')
f = numpy.array([100.0])
if dospectral:
flux = numpy.array(
[f * numpy.power(freq / 1e8, -0.7) for freq in self.frequency])
else:
flux = numpy.array([f])
self.phasecentre = SkyCoord(ra=+180.0 * u.deg,
dec=-60.0 * u.deg,
frame='icrs',
equinox='J2000')
self.blockvis_list = [
ingest_unittest_visibility(self.low, [self.frequency[i]],
[self.channelwidth[i]],
self.times,
self.vis_pol,
self.phasecentre,
block=True,
zerow=zerow) for i in range(nfreqwin)
]
for v in self.blockvis_list:
v.data['vis'][...] = 1.0 + 0.0j
self.error_blockvis_list = [
copy_visibility(v) for v in self.blockvis_list
]
gt = create_gaintable_from_blockvisibility(self.blockvis_list[0])
gt = simulate_gaintable(gt,
phase_error=0.1,
amplitude_error=0.0,
smooth_channels=1,
leakage=0.0)
self.error_blockvis_list = [
apply_gaintable(self.error_blockvis_list[i], gt)
for i in range(self.freqwin)
]
assert numpy.max(
numpy.abs(self.error_blockvis_list[0].vis -
self.blockvis_list[0].vis)) > 0.0
def create_blockvisibility_iterator(
config: Configuration,
times: numpy.array,
frequency: numpy.array,
channel_bandwidth,
phasecentre: SkyCoord,
weight: float = 1,
polarisation_frame=PolarisationFrame('stokesI'),
integration_time=1.0,
number_integrations=1,
predict=predict_2d,
model=None,
components=None,
phase_error=0.0,
amplitude_error=0.0,
sleep=0.0,
**kwargs):
""" Create a sequence of Visibilities and optionally predicting and coalescing
This is useful mainly for performing large simulations. Do something like::
vis_iter = create_blockvisibility_iterator(config, times, frequency, channel_bandwidth, phasecentre=phasecentre,
weight=1.0, integration_time=30.0, number_integrations=3)
for i, vis in enumerate(vis_iter):
if i == 0:
fullvis = vis
else:
fullvis = append_visibility(fullvis, vis)
:param config: Configuration of antennas
:param times: hour angles in radians
:param frequency: frequencies (Hz] Shape [nchan]
:param weight: weight of a single sample
:param phasecentre: phasecentre of observation
:param npol: Number of polarizations
:param integration_time: Integration time ('auto' or value in s)
:param number_integrations: Number of integrations to be created at each time.
:param model: Model image to be inserted
:param components: Components to be inserted
:param sleep_time: Time to sleep between yields
:return: Visibility
"""
for time in times:
actualtimes = time + numpy.arange(
0, number_integrations) * integration_time * numpy.pi / 43200.0
bvis = create_blockvisibility(config,
actualtimes,
frequency=frequency,
phasecentre=phasecentre,
weight=weight,
polarisation_frame=polarisation_frame,
integration_time=integration_time,
channel_bandwidth=channel_bandwidth)
if model is not None:
vis = convert_blockvisibility_to_visibility(bvis)
vis = predict(vis, model, **kwargs)
bvis = convert_visibility_to_blockvisibility(vis)
if components is not None:
vis = dft_skycomponent_visibility(bvis, components)
# Add phase errors
if phase_error > 0.0 or amplitude_error > 0.0:
gt = create_gaintable_from_blockvisibility(bvis)
gt = simulate_gaintable(gt=gt,
phase_error=phase_error,
amplitude_error=amplitude_error)
bvis = apply_gaintable(bvis, gt)
import time
time.sleep(sleep)
yield bvis
def actualSetUp(self, add_errors=False, nfreqwin=7, dospectral=True, dopol=False, zerow=True):
self.npixel = 512
self.low = create_named_configuration('LOWBD2', rmax=750.0)
self.freqwin = nfreqwin
self.vis_list = list()
self.ntimes = 5
self.times = numpy.linspace(-3.0, +3.0, self.ntimes) * numpy.pi / 12.0
self.frequency = numpy.linspace(0.8e8, 1.2e8, self.freqwin)
if self.freqwin > 1:
self.channelwidth = numpy.array(self.freqwin * [self.frequency[1] - self.frequency[0]])
else:
self.channelwidth = numpy.array([1e6])
if dopol:
self.vis_pol = PolarisationFrame('linear')
self.image_pol = PolarisationFrame('stokesIQUV')
f = numpy.array([100.0, 20.0, 0.0, 0.0])
else:
self.vis_pol = PolarisationFrame('stokesI')
self.image_pol = PolarisationFrame('stokesI')
f = numpy.array([100.0])
if dospectral:
flux = numpy.array([f * numpy.power(freq / 1e8, -0.7) for freq in self.frequency])
else:
flux = numpy.array([f])
self.phasecentre = SkyCoord(ra=+180.0 * u.deg, dec=-60.0 * u.deg, frame='icrs', equinox='J2000')
self.blockvis_list = [ingest_unittest_visibility(self.low,
[self.frequency[i]],
[self.channelwidth[i]],
self.times,
self.vis_pol,
self.phasecentre, block=True,
zerow=zerow)
for i in range(nfreqwin)]
self.vis_list = [convert_blockvisibility_to_visibility(bv) for bv in self.blockvis_list]
self.model_imagelist = [
create_unittest_model(self.vis_list[i], self.image_pol, npixel=self.npixel, cellsize=0.0005)
for i in range(nfreqwin)]
self.components_list = [create_unittest_components(self.model_imagelist[freqwin],
flux[freqwin, :][numpy.newaxis, :])
for freqwin, m in enumerate(self.model_imagelist)]
self.blockvis_list = [
dft_skycomponent_visibility(self.blockvis_list[freqwin], self.components_list[freqwin])
for freqwin, _ in enumerate(self.blockvis_list)]
self.model_imagelist = [insert_skycomponent(self.model_imagelist[freqwin], self.components_list[freqwin])
for freqwin in range(nfreqwin)]
model = self.model_imagelist[0]
self.cmodel = smooth_image(model)
if self.persist:
export_image_to_fits(model, '%s/test_imaging_serial_model.fits' % self.dir)
export_image_to_fits(self.cmodel, '%s/test_imaging_serial_cmodel.fits' % self.dir)
if add_errors:
gt = create_gaintable_from_blockvisibility(self.blockvis_list[0])
gt = simulate_gaintable(gt, phase_error=0.1, amplitude_error=0.0, smooth_channels=1, leakage=0.0)
self.blockvis_list = [apply_gaintable(self.blockvis_list[i], gt)
for i in range(self.freqwin)]
self.vis_list = [convert_blockvisibility_to_visibility(bv) for bv in self.blockvis_list]
self.model_imagelist = [
create_unittest_model(self.vis_list[i], self.image_pol, npixel=self.npixel, cellsize=0.0005)
for i in range(nfreqwin)]
def ingest_visibility(self,
freq=None,
chan_width=None,
times=None,
add_errors=False,
block=True,
bandpass=False):
if freq is None:
freq = [1e8]
if chan_width is None:
chan_width = [1e6]
if times is None:
times = (numpy.pi / 12.0) * numpy.linspace(-3.0, 3.0, 5)
lowcore = create_named_configuration('LOWBD2', rmax=750.0)
frequency = numpy.array(freq)
channel_bandwidth = numpy.array(chan_width)
phasecentre = SkyCoord(ra=+180.0 * u.deg,
dec=-60.0 * u.deg,
frame='icrs',
equinox='J2000')
if block:
vt = create_blockvisibility(
lowcore,
times,
frequency,
channel_bandwidth=channel_bandwidth,
weight=1.0,
phasecentre=phasecentre,
polarisation_frame=PolarisationFrame("stokesI"))
else:
vt = create_visibility(
lowcore,
times,
frequency,
channel_bandwidth=channel_bandwidth,
weight=1.0,
phasecentre=phasecentre,
polarisation_frame=PolarisationFrame("stokesI"))
cellsize = 0.001
model = create_image_from_visibility(
vt,
npixel=self.npixel,
cellsize=cellsize,
npol=1,
frequency=frequency,
phasecentre=phasecentre,
polarisation_frame=PolarisationFrame("stokesI"))
nchan = len(self.frequency)
flux = numpy.array(nchan * [[100.0]])
facets = 4
rpix = model.wcs.wcs.crpix - 1.0
spacing_pixels = self.npixel // facets
centers = [-1.5, -0.5, 0.5, 1.5]
comps = list()
for iy in centers:
for ix in centers:
p = int(round(rpix[0] + ix * spacing_pixels * numpy.sign(model.wcs.wcs.cdelt[0]))), \
int(round(rpix[1] + iy * spacing_pixels * numpy.sign(model.wcs.wcs.cdelt[1])))
sc = pixel_to_skycoord(p[0], p[1], model.wcs, origin=1)
comp = create_skycomponent(
direction=sc,
flux=flux,
frequency=frequency,
polarisation_frame=PolarisationFrame("stokesI"))
comps.append(comp)
if block:
dft_skycomponent_visibility(vt, comps)
else:
dft_skycomponent_visibility(vt, comps)
insert_skycomponent(model, comps)
self.comps = comps
self.model = copy_image(model)
self.empty_model = create_empty_image_like(model)
export_image_to_fits(
model, '%s/test_pipeline_functions_model.fits' % (self.dir))
if add_errors:
# These will be the same for all calls
numpy.random.seed(180555)
gt = create_gaintable_from_blockvisibility(vt)
gt = simulate_gaintable(gt, phase_error=1.0, amplitude_error=0.0)
vt = apply_gaintable(vt, gt)
if bandpass:
bgt = create_gaintable_from_blockvisibility(vt, timeslice=1e5)
bgt = simulate_gaintable(bgt,
phase_error=0.01,
amplitude_error=0.01,
smooth_channels=4)
vt = apply_gaintable(vt, bgt)
return vt
