def test_readwriteskymodel_no_image(self):
vis = create_blockvisibility(
self.midcore,
self.times,
self.frequency,
channel_bandwidth=self.channel_bandwidth,
phasecentre=self.phasecentre,
polarisation_frame=PolarisationFrame("linear"),
weight=1.0)
gt = create_gaintable_from_blockvisibility(vis, timeslice='auto')
gt = simulate_gaintable(gt, phase_error=1.0, amplitude_error=0.1)
sm = SkyModel(components=[self.comp], gaintable=gt)
config = {
"buffer": {
"directory": self.dir
},
"skymodel": {
"name": "test_bufferskymodel.hdf",
"data_model": "SkyModel"
}
}
bdm = BufferSkyModel(config["buffer"], config["skymodel"], sm)
bdm.sync()
new_bdm = BufferSkyModel(config["buffer"], config["skymodel"])
new_bdm.sync()
newsm = bdm.memory_data_model
assert newsm.components[0].flux.shape == self.comp.flux.shape
assert newsm.gaintable.data.shape == gt.data.shape
def test_readwritegaintable(self):
self.vis = create_blockvisibility(
self.midcore,
self.times,
self.frequency,
channel_bandwidth=self.channel_bandwidth,
phasecentre=self.phasecentre,
polarisation_frame=PolarisationFrame("linear"),
weight=1.0)
gt = create_gaintable_from_blockvisibility(self.vis, timeslice='auto')
gt = simulate_gaintable(gt, phase_error=1.0, amplitude_error=0.1)
config = {
"buffer": {
"directory": self.dir
},
"gaintable": {
"name": "test_buffergaintable.hdf",
"data_model": "GainTable"
}
}
bdm = BufferGainTable(config["buffer"], config["gaintable"], gt)
bdm.sync()
new_bdm = BufferGainTable(config["buffer"], config["gaintable"])
new_bdm.sync()
newgt = bdm.memory_data_model
assert gt.data.shape == newgt.data.shape
assert numpy.max(numpy.abs(gt.gain - newgt.gain)) < 1e-15
def test_plot_gaintable_scalar(self):
self.actualSetup('stokesI', 'stokesI', f=[100.0])
gt = create_gaintable_from_blockvisibility(self.vis)
log.info("Created gain table: %s" % (gaintable_summary(gt)))
gt = simulate_gaintable(gt, phase_error=0.1, amplitude_error=0.1)
plt.clf()
gaintable_plot(gt)
plt.show(block=False)
def test_apply_gaintable_only(self):
for spf, dpf in[('stokesI', 'stokesI'), ('stokesIQUV', 'linear'), ('stokesIQUV', 'circular')]:
self.actualSetup(spf, dpf)
gt = create_gaintable_from_blockvisibility(self.vis, timeslice='auto')
log.info("Created gain table: %s" % (gaintable_summary(gt)))
gt = simulate_gaintable(gt, phase_error=0.1, amplitude_error=0.01)
original = copy_visibility(self.vis)
vis = apply_gaintable(self.vis, gt)
error = numpy.max(numpy.abs(vis.vis - original.vis))
assert error > 10.0, "Error = %f" % (error)
def test_solve_gaintable_scalar_timeslice(self):
self.actualSetup('stokesI', 'stokesI', f=[100.0], ntimes=10)
gt = create_gaintable_from_blockvisibility(self.vis, timeslice=120.0)
log.info("Created gain table: %s" % (gaintable_summary(gt)))
gt = simulate_gaintable(gt, phase_error=10.0, amplitude_error=0.0)
original = copy_visibility(self.vis)
self.vis = apply_gaintable(self.vis, gt)
gtsol = solve_gaintable(self.vis, original, phase_only=True, niter=200)
residual = numpy.max(gtsol.residual)
assert residual < 3e-8, "Max residual = %s" % (residual)
assert numpy.max(numpy.abs(gtsol.gain - 1.0)) > 0.1
def test_solve_gaintable_stokesI_pointsource(self):
self.actualSetup('stokesI', 'stokesI', f=[100.0])
gt = create_gaintable_from_blockvisibility(self.vis)
log.info("Created gain table: %s" % (gaintable_summary(gt)))
gt = simulate_gaintable(gt, phase_error=10.0, amplitude_error=0.0)
original = copy_visibility(self.vis)
self.vis = apply_gaintable(self.vis, gt)
point_vis = divide_visibility(self.vis, original)
gtsol = solve_gaintable(point_vis, phase_only=False, niter=200)
residual = numpy.max(gtsol.residual)
assert residual < 3e-8, "Max residual = %s" % (residual)
assert numpy.max(numpy.abs(gtsol.gain - 1.0)) > 0.1
def test_create_gaintable_from_visibility_interval(self):
for timeslice in [10.0, 'auto', 1e5]:
for spf, dpf in[('stokesIQUV', 'linear')]:
self.actualSetup(spf, dpf)
gt = create_gaintable_from_blockvisibility(self.vis, timeslice=timeslice)
log.info("Created gain table: %s" % (gaintable_summary(gt)))
gt = simulate_gaintable(gt, phase_error=1.0)
original = copy_visibility(self.vis)
vis = apply_gaintable(self.vis, gt)
assert numpy.max(numpy.abs(original.vis)) > 0.0
assert numpy.max(numpy.abs(vis.vis)) > 0.0
assert numpy.max(numpy.abs(vis.vis - original.vis)) > 0.0
def test_solve_gaintable_stokesI_small_n_large_t(self):
# Select only 6 stations
self.actualSetup('stokesI', 'stokesI', f=[100.0], ntimes=4000, rmax=83)
gt = create_gaintable_from_blockvisibility(self.vis)
log.info("Created gain table: %s" % (gaintable_summary(gt)))
gt = simulate_gaintable(gt, phase_error=10.0, amplitude_error=0.0)
gt.data['gain'] = gt.gain[1, ...]
original = copy_visibility(self.vis)
self.vis = apply_gaintable(self.vis, gt)
gtsol = solve_gaintable(self.vis, original, phase_only=True, niter=200)
self.vis = apply_gaintable(self.vis, gtsol)
residual = numpy.max(gtsol.residual)
assert residual < 3e-8, "Max residual = %s" % (residual)
assert numpy.max(numpy.abs(gtsol.gain - 1.0)) > 0.1
def corrupt_vis(vis, gt, **kwargs):
if isinstance(vis, Visibility):
bv = convert_visibility_to_blockvisibility(vis)
else:
bv = vis
if gt is None:
gt = create_gaintable_from_blockvisibility(bv, **kwargs)
gt = simulate_gaintable(gt, **kwargs)
bv = apply_gaintable(bv, gt)
if isinstance(vis, Visibility):
return convert_blockvisibility_to_visibility(bv)
else:
return bv
def test_readwritegaintable(self):
self.vis = create_blockvisibility(self.mid, self.times, self.frequency,
channel_bandwidth=self.channel_bandwidth,
phasecentre=self.phasecentre,
polarisation_frame=PolarisationFrame("linear"),
weight=1.0)
gt = create_gaintable_from_blockvisibility(self.vis, timeslice='auto')
gt = simulate_gaintable(gt, phase_error=1.0, amplitude_error=0.1)
export_gaintable_to_hdf5(gt, '%s/test_data_model_helpers_gaintable.hdf' % self.dir)
newgt = import_gaintable_from_hdf5('%s/test_data_model_helpers_gaintable.hdf' % self.dir)
for key in gt.data.dtype.fields:
assert numpy.max(numpy.abs(newgt.data[key] - gt.data[key])) < 1e-15
assert gt.data.shape == newgt.data.shape
assert numpy.max(numpy.abs(gt.gain - newgt.gain)) < 1e-15
def test_solve_gaintable_scalar_normalise(self):
self.actualSetup('stokesI', 'stokesI', f=[100.0])
gt = create_gaintable_from_blockvisibility(self.vis)
log.info("Created gain table: %s" % (gaintable_summary(gt)))
gt = simulate_gaintable(gt, phase_error=0.0, amplitude_error=0.1)
gt.data['gain'] *= 2.0
original = copy_visibility(self.vis)
self.vis = apply_gaintable(self.vis, gt)
gtsol = solve_gaintable(self.vis,
original,
phase_only=False,
niter=200,
normalise_gains=True)
residual = numpy.max(gtsol.residual)
assert residual < 3e-8, "Max residual = %s" % (residual)
assert numpy.max(numpy.abs(gtsol.gain - 1.0)) > 0.1
def test_calibrate_G_function(self):
self.actualSetup('stokesIQUV', 'linear', f=[100.0, 0.0, 0.0, 50.0])
# Prepare the corrupted visibility data_models
gt = create_gaintable_from_blockvisibility(self.vis)
log.info("Created gain table: %s" % (gaintable_summary(gt)))
gt = simulate_gaintable(gt, phase_error=0.0, amplitude_error=0.1)
original = copy_visibility(self.vis)
self.vis = apply_gaintable(self.vis, gt)
# Now get the control dictionary and calibrate
controls = create_calibration_controls()
controls['G']['first_selfcal'] = 0
calibrated_vis, gaintables = calibrate_chain(self.vis,
original,
calibration_context='G',
controls=controls)
residual = numpy.max(gaintables['G'].residual)
assert residual < 1e-8, "Max T residual = %s" % residual
def test_solve_gaintable_stokesI_bandpass(self):
self.actualSetup('stokesI', 'stokesI', f=[100.0], vnchan=128)
gt = create_gaintable_from_blockvisibility(self.vis)
log.info("Created gain table: %s" % (gaintable_summary(gt)))
gt = simulate_gaintable(gt,
phase_error=10.0,
amplitude_error=0.01,
smooth_channels=8)
original = copy_visibility(self.vis)
self.vis = apply_gaintable(self.vis, gt)
gtsol = solve_gaintable(self.vis,
original,
phase_only=False,
niter=200,
damping=0.5)
residual = numpy.max(gtsol.residual)
assert residual < 3e-8, "Max residual = %s" % (residual)
assert numpy.max(numpy.abs(gtsol.gain - 1.0)) > 0.1
def core_solve(self,
spf,
dpf,
phase_error=0.1,
amplitude_error=0.0,
leakage=0.0,
phase_only=True,
niter=200,
crosspol=False,
residual_tol=1e-6,
f=None,
vnchan=3,
timeslice='auto'):
if f is None:
f = [100.0, 50.0, -10.0, 40.0]
self.actualSetup(spf, dpf, f=f, vnchan=vnchan)
gt = create_gaintable_from_blockvisibility(self.vis,
timeslice=timeslice)
log.info("Created gain table: %s" % (gaintable_summary(gt)))
gt = simulate_gaintable(gt,
phase_error=phase_error,
amplitude_error=amplitude_error,
leakage=leakage)
original = copy_visibility(self.vis)
vis = apply_gaintable(self.vis, gt)
gtsol = solve_gaintable(self.vis,
original,
phase_only=phase_only,
niter=niter,
crosspol=crosspol,
tol=1e-6)
vis = apply_gaintable(vis, gtsol, inverse=True)
residual = numpy.max(gtsol.residual)
assert residual < residual_tol, "%s %s Max residual = %s" % (spf, dpf,
residual)
log.debug(qa_gaintable(gt))
assert numpy.max(numpy.abs(gtsol.gain - 1.0)) > 0.1
def actualSetUp(self,
nfreqwin=3,
dospectral=True,
dopol=False,
amp_errors=None,
phase_errors=None,
zerow=True):
if amp_errors is None:
amp_errors = {'T': 0.0, 'G': 0.1}
if phase_errors is None:
phase_errors = {'T': 1.0, 'G': 0.0}
self.npixel = 512
self.low = create_named_configuration('LOWBD2', rmax=750.0)
self.freqwin = nfreqwin
self.vis_list = list()
self.ntimes = 1
self.times = numpy.linspace(-3.0, +3.0, self.ntimes) * numpy.pi / 12.0
self.frequency = numpy.linspace(0.8e8, 1.2e8, self.freqwin)
if self.freqwin > 1:
self.channelwidth = numpy.array(
self.freqwin * [self.frequency[1] - self.frequency[0]])
else:
self.channelwidth = numpy.array([1e6])
if dopol:
self.vis_pol = PolarisationFrame('linear')
self.image_pol = PolarisationFrame('stokesIQUV')
f = numpy.array([100.0, 20.0, -10.0, 1.0])
else:
self.vis_pol = PolarisationFrame('stokesI')
self.image_pol = PolarisationFrame('stokesI')
f = numpy.array([100.0])
if dospectral:
flux = numpy.array(
[f * numpy.power(freq / 1e8, -0.7) for freq in self.frequency])
else:
flux = numpy.array([f])
self.phasecentre = SkyCoord(ra=+180.0 * u.deg,
dec=-60.0 * u.deg,
frame='icrs',
equinox='J2000')
self.blockvis_list = [
ingest_unittest_visibility(self.low, [self.frequency[i]],
[self.channelwidth[i]],
self.times,
self.vis_pol,
self.phasecentre,
block=True,
zerow=zerow) for i in range(nfreqwin)
]
for v in self.blockvis_list:
v.data['vis'][...] = 1.0 + 0.0j
self.error_blockvis_list = [
copy_visibility(v) for v in self.blockvis_list
]
gt = create_gaintable_from_blockvisibility(self.blockvis_list[0])
gt = simulate_gaintable(gt,
phase_error=0.1,
amplitude_error=0.0,
smooth_channels=1,
leakage=0.0)
self.error_blockvis_list = [
apply_gaintable(self.error_blockvis_list[i], gt)
for i in range(self.freqwin)
]
assert numpy.max(
numpy.abs(self.error_blockvis_list[0].vis -
self.blockvis_list[0].vis)) > 0.0
def actualSetUp(self,
add_errors=False,
nfreqwin=7,
dospectral=True,
dopol=False,
zerow=True):
self.npixel = 512
self.low = create_named_configuration('LOWBD2', rmax=750.0)
self.freqwin = nfreqwin
self.vis_list = list()
self.ntimes = 5
self.times = numpy.linspace(-3.0, +3.0, self.ntimes) * numpy.pi / 12.0
self.frequency = numpy.linspace(0.8e8, 1.2e8, self.freqwin)
if self.freqwin > 1:
self.channelwidth = numpy.array(
self.freqwin * [self.frequency[1] - self.frequency[0]])
else:
self.channelwidth = numpy.array([1e6])
if dopol:
self.vis_pol = PolarisationFrame('linear')
self.image_pol = PolarisationFrame('stokesIQUV')
f = numpy.array([100.0, 20.0, 0.0, 0.0])
else:
self.vis_pol = PolarisationFrame('stokesI')
self.image_pol = PolarisationFrame('stokesI')
f = numpy.array([100.0])
if dospectral:
flux = numpy.array(
[f * numpy.power(freq / 1e8, -0.7) for freq in self.frequency])
else:
flux = numpy.array([f])
self.phasecentre = SkyCoord(ra=+180.0 * u.deg,
dec=-60.0 * u.deg,
frame='icrs',
equinox='J2000')
self.blockvis_list = \
[rsexecute.execute(ingest_unittest_visibility, nout=1)(self.low,
[self.frequency[i]],
[self.channelwidth[i]],
self.times,
self.vis_pol,
self.phasecentre, block=True,
zerow=zerow)
for i in range(nfreqwin)]
self.blockvis_list = rsexecute.compute(self.blockvis_list, sync=True)
self.blockvis_list = rsexecute.scatter(self.blockvis_list)
self.vis_list = [
rsexecute.execute(convert_blockvisibility_to_visibility,
nout=1)(bv) for bv in self.blockvis_list
]
self.vis_list = rsexecute.compute(self.vis_list, sync=True)
self.vis_list = rsexecute.scatter(self.vis_list)
self.model_imagelist = [
rsexecute.execute(create_unittest_model,
nout=1)(self.vis_list[i],
self.image_pol,
npixel=self.npixel,
cellsize=0.0005) for i in range(nfreqwin)
]
self.model_imagelist = rsexecute.compute(self.model_imagelist,
sync=True)
self.model_imagelist = rsexecute.scatter(self.model_imagelist)
self.components_list = [
rsexecute.execute(create_unittest_components)(
self.model_imagelist[freqwin],
flux[freqwin, :][numpy.newaxis, :])
for freqwin, m in enumerate(self.model_imagelist)
]
self.components_list = rsexecute.compute(self.components_list,
sync=True)
self.components_list = rsexecute.scatter(self.components_list)
self.blockvis_list = [
rsexecute.execute(dft_skycomponent_visibility)(
self.blockvis_list[freqwin], self.components_list[freqwin])
for freqwin, _ in enumerate(self.blockvis_list)
]
self.blockvis_list = rsexecute.compute(self.blockvis_list, sync=True)
self.vis = self.blockvis_list[0]
self.blockvis_list = rsexecute.scatter(self.blockvis_list)
self.model_imagelist = [
rsexecute.execute(insert_skycomponent,
nout=1)(self.model_imagelist[freqwin],
self.components_list[freqwin])
for freqwin in range(nfreqwin)
]
self.model_imagelist = rsexecute.compute(self.model_imagelist,
sync=True)
model = self.model_imagelist[0]
self.cmodel = smooth_image(model)
if self.persist:
export_image_to_fits(
model, '%s/test_pipelines_rsexecute_model.fits' % self.dir)
export_image_to_fits(
self.cmodel,
'%s/test_pipelines_rsexecute_cmodel.fits' % self.dir)
if add_errors:
gt = create_gaintable_from_blockvisibility(self.vis)
gt = simulate_gaintable(gt,
phase_error=0.1,
amplitude_error=0.0,
smooth_channels=1,
leakage=0.0)
self.blockvis_list = [
rsexecute.execute(apply_gaintable,
nout=1)(self.blockvis_list[i], gt)
for i in range(self.freqwin)
]
self.blockvis_list = rsexecute.compute(self.blockvis_list,
sync=True)
self.blockvis_list = rsexecute.scatter(self.blockvis_list)
self.vis_list = [
rsexecute.execute(convert_blockvisibility_to_visibility)(bv)
for bv in self.blockvis_list
]
self.vis_list = rsexecute.compute(self.vis_list, sync=True)
self.vis_list = rsexecute.scatter(self.vis_list)
self.model_imagelist = [
rsexecute.execute(create_unittest_model,
nout=1)(self.vis_list[i],
self.image_pol,
npixel=self.npixel,
cellsize=0.0005) for i in range(nfreqwin)
]
self.model_imagelist = rsexecute.compute(self.model_imagelist,
sync=True)
self.model_imagelist = rsexecute.scatter(self.model_imagelist)
def ingest_visibility(self,
freq=None,
chan_width=None,
times=None,
add_errors=False,
block=True,
bandpass=False):
if freq is None:
freq = [1e8]
if chan_width is None:
chan_width = [1e6]
if times is None:
times = (numpy.pi / 12.0) * numpy.linspace(-3.0, 3.0, 5)
lowcore = create_named_configuration('LOWBD2', rmax=750.0)
frequency = numpy.array(freq)
channel_bandwidth = numpy.array(chan_width)
phasecentre = SkyCoord(ra=+180.0 * u.deg,
dec=-60.0 * u.deg,
frame='icrs',
equinox='J2000')
if block:
vt = create_blockvisibility(
lowcore,
times,
frequency,
channel_bandwidth=channel_bandwidth,
weight=1.0,
phasecentre=phasecentre,
polarisation_frame=PolarisationFrame("stokesI"))
else:
vt = create_visibility(
lowcore,
times,
frequency,
channel_bandwidth=channel_bandwidth,
weight=1.0,
phasecentre=phasecentre,
polarisation_frame=PolarisationFrame("stokesI"))
cellsize = 0.001
model = create_image_from_visibility(
vt,
npixel=self.npixel,
cellsize=cellsize,
npol=1,
frequency=frequency,
phasecentre=phasecentre,
polarisation_frame=PolarisationFrame("stokesI"))
nchan = len(self.frequency)
flux = numpy.array(nchan * [[100.0]])
facets = 4
rpix = model.wcs.wcs.crpix - 1.0
spacing_pixels = self.npixel // facets
centers = [-1.5, -0.5, 0.5, 1.5]
comps = list()
for iy in centers:
for ix in centers:
p = int(round(rpix[0] + ix * spacing_pixels * numpy.sign(model.wcs.wcs.cdelt[0]))), \
int(round(rpix[1] + iy * spacing_pixels * numpy.sign(model.wcs.wcs.cdelt[1])))
sc = pixel_to_skycoord(p[0], p[1], model.wcs, origin=1)
comp = create_skycomponent(
direction=sc,
flux=flux,
frequency=frequency,
polarisation_frame=PolarisationFrame("stokesI"))
comps.append(comp)
if block:
dft_skycomponent_visibility(vt, comps)
else:
dft_skycomponent_visibility(vt, comps)
insert_skycomponent(model, comps)
self.comps = comps
self.model = copy_image(model)
self.empty_model = create_empty_image_like(model)
export_image_to_fits(
model, '%s/test_pipeline_functions_model.fits' % (self.dir))
if add_errors:
# These will be the same for all calls
numpy.random.seed(180555)
gt = create_gaintable_from_blockvisibility(vt)
gt = simulate_gaintable(gt, phase_error=1.0, amplitude_error=0.0)
vt = apply_gaintable(vt, gt)
if bandpass:
bgt = create_gaintable_from_blockvisibility(vt, timeslice=1e5)
bgt = simulate_gaintable(bgt,
phase_error=0.01,
amplitude_error=0.01,
smooth_channels=4)
vt = apply_gaintable(vt, bgt)
return vt
