def test_predict(self):
self.actualSetUp(zerow=True)
self.skymodel_list = [
arlexecute.execute(create_low_test_skymodel_from_gleam)(
npixel=self.npixel,
cellsize=self.cellsize,
frequency=[self.frequency[f]],
phasecentre=self.phasecentre,
polarisation_frame=PolarisationFrame("stokesI"),
flux_limit=0.3,
flux_threshold=0.3,
flux_max=5.0) for f, freq in enumerate(self.frequency)
]
self.skymodel_list = arlexecute.compute(self.skymodel_list, sync=True)
assert isinstance(self.skymodel_list[0].images[0],
Image), self.skymodel_list[0].images[0]
assert isinstance(self.skymodel_list[0].components[0],
Skycomponent), self.skymodel_list[0].components[0]
assert len(self.skymodel_list[0].components) == 119, len(
self.skymodel_list[0].components)
assert len(self.skymodel_list[0].images) == 1, len(
self.skymodel_list[0].images)
assert numpy.max(numpy.abs(
self.skymodel_list[0].images[0].data)) > 0.0, "Image is empty"
self.skymodel_list = arlexecute.scatter(self.skymodel_list)
skymodel_vislist = predict_skymodel_list_arlexecute_workflow(
self.vis_list, self.skymodel_list, context='2d')
skymodel_vislist = arlexecute.compute(skymodel_vislist, sync=True)
assert numpy.max(numpy.abs(skymodel_vislist[0].vis)) > 0.0
def test_zero_list(self):
self.actualSetUp()
centre = self.freqwin // 2
vis_list = zero_list_arlexecute_workflow(self.vis_list)
vis_list = arlexecute.compute(vis_list, sync=True)
assert numpy.max(numpy.abs(vis_list[centre].vis)) < 1e-15, numpy.max(
numpy.abs(vis_list[centre].vis))
predicted_vis_list = [
arlexecute.execute(predict_skycomponent_visibility)(
vis_list[freqwin], self.components_list[freqwin])
for freqwin, _ in enumerate(self.frequency)
]
predicted_vis_list = arlexecute.compute(predicted_vis_list, sync=True)
assert numpy.max(numpy.abs(predicted_vis_list[centre].vis)) > 0.0, \
numpy.max(numpy.abs(predicted_vis_list[centre].vis))
diff_vis_list = subtract_list_arlexecute_workflow(
self.vis_list, predicted_vis_list)
diff_vis_list = arlexecute.compute(diff_vis_list, sync=True)
assert numpy.max(numpy.abs(
diff_vis_list[centre].vis)) < 1e-15, numpy.max(
numpy.abs(diff_vis_list[centre].vis))
def test_crosssubtract_datamodel(self):
self.actualSetUp(zerow=True)
future_vis = arlexecute.scatter(self.vis_list[0])
future_skymodel_list = arlexecute.scatter(self.skymodel_list)
skymodel_vislist = predict_skymodel_list_arlexecute_workflow(
future_vis, future_skymodel_list, context='2d', docal=True)
skymodel_vislist = arlexecute.compute(skymodel_vislist, sync=True)
vobs = sum_predict_results(skymodel_vislist)
future_vobs = arlexecute.scatter(vobs)
skymodel_vislist = crosssubtract_datamodels_skymodel_list_arlexecute_workflow(
future_vobs, skymodel_vislist)
skymodel_vislist = arlexecute.compute(skymodel_vislist, sync=True)
result_skymodel = [
SkyModel(components=None, image=self.skymodel_list[-1].image)
for v in skymodel_vislist
]
self.vis_list = arlexecute.scatter(self.vis_list)
result_skymodel = invert_skymodel_list_arlexecute_workflow(
skymodel_vislist, result_skymodel, context='2d', docal=True)
results = arlexecute.compute(result_skymodel, sync=True)
assert numpy.max(numpy.abs(results[0][0].data)) > 0.0
assert numpy.max(numpy.abs(results[0][1])) > 0.0
if self.plot:
import matplotlib.pyplot as plt
from wrappers.arlexecute.image.operations import show_image
show_image(results[0][0],
title='Dirty image after cross-subtraction',
vmax=0.1,
vmin=-0.01)
plt.show()
def actualSetUp(self, freqwin=1, block=True, dopol=False, zerow=False):
self.npixel = 1024
self.low = create_named_configuration('LOWBD2', rmax=550.0)
self.freqwin = freqwin
self.blockvis_list = list()
self.ntimes = 5
self.cellsize = 0.0005
# Choose the interval so that the maximum change in w is smallish
integration_time = numpy.pi * (24 / (12 * 60))
self.times = numpy.linspace(-integration_time * (self.ntimes // 2), integration_time * (self.ntimes // 2),
self.ntimes)
if freqwin > 1:
self.frequency = numpy.linspace(0.8e8, 1.2e8, self.freqwin)
self.channelwidth = numpy.array(freqwin * [self.frequency[1] - self.frequency[0]])
else:
self.frequency = numpy.array([1.0e8])
self.channelwidth = numpy.array([4e7])
if dopol:
self.vis_pol = PolarisationFrame('linear')
self.image_pol = PolarisationFrame('stokesIQUV')
f = numpy.array([100.0, 20.0, -10.0, 1.0])
else:
self.vis_pol = PolarisationFrame('stokesI')
self.image_pol = PolarisationFrame('stokesI')
f = numpy.array([100.0])
self.phasecentre = SkyCoord(ra=+0.0 * u.deg, dec=-40.0 * u.deg, frame='icrs', equinox='J2000')
self.blockvis_list = [arlexecute.execute(ingest_unittest_visibility)(self.low,
[self.frequency[freqwin]],
[self.channelwidth[freqwin]],
self.times,
self.vis_pol,
self.phasecentre, block=block,
zerow=zerow)
for freqwin, _ in enumerate(self.frequency)]
self.blockvis_list = arlexecute.compute(self.blockvis_list, sync=True)
self.vis_list = [arlexecute.execute(convert_blockvisibility_to_visibility)(bv) for bv in self.blockvis_list]
self.vis_list = arlexecute.compute(self.vis_list, sync=True)
self.skymodel_list = [arlexecute.execute(create_low_test_skymodel_from_gleam)
(npixel=self.npixel, cellsize=self.cellsize, frequency=[self.frequency[f]],
phasecentre=self.phasecentre,
polarisation_frame=PolarisationFrame("stokesI"),
flux_limit=0.6,
flux_threshold=1.0,
flux_max=5.0) for f, freq in enumerate(self.frequency)]
self.skymodel_list = arlexecute.compute(self.skymodel_list, sync=True)
assert isinstance(self.skymodel_list[0].image, Image), self.skymodel_list[0].image
assert isinstance(self.skymodel_list[0].components[0], Skycomponent), self.skymodel_list[0].components[0]
assert len(self.skymodel_list[0].components) == 19, len(self.skymodel_list[0].components)
self.skymodel_list = expand_skymodel_by_skycomponents(self.skymodel_list[0])
assert len(self.skymodel_list) == 20, len(self.skymodel_list)
assert numpy.max(numpy.abs(self.skymodel_list[-1].image.data)) > 0.0, "Image is empty"
self.vis_list = [copy_visibility(self.vis_list[0], zero=True) for i, _ in enumerate(self.skymodel_list)]
def test_create_simulate_vis_list(self):
vis_list = simulate_list_arlexecute_workflow(
frequency=self.frequency, channel_bandwidth=self.channel_bandwidth)
assert len(vis_list) == len(self.frequency)
vt = arlexecute.compute(vis_list[0], sync=True)
assert isinstance(vt, BlockVisibility)
assert vt.nvis > 0
def test_deconvolve_spectral(self):
self.actualSetUp(add_errors=True)
dirty_imagelist = invert_list_arlexecute_workflow(self.vis_list,
self.model_imagelist,
context='2d',
dopsf=False,
normalize=True)
psf_imagelist = invert_list_arlexecute_workflow(self.vis_list,
self.model_imagelist,
context='2d',
dopsf=True,
normalize=True)
dirty_imagelist = arlexecute.persist(dirty_imagelist)
psf_imagelist = arlexecute.persist(psf_imagelist)
deconvolved = deconvolve_list_arlexecute_workflow(
dirty_imagelist,
psf_imagelist,
self.model_imagelist,
niter=1000,
fractional_threshold=0.1,
scales=[0, 3, 10],
threshold=0.1,
gain=0.7)
deconvolved = arlexecute.persist(deconvolved)
deconvolved = arlexecute.compute(deconvolved, sync=True)
export_image_to_fits(
deconvolved[0], '%s/test_imaging_%s_deconvolve_spectral.fits' %
(self.dir, arlexecute.type()))
def test_continuum_imaging_pipeline(self):
self.actualSetUp(add_errors=False, block=True)
continuum_imaging_list = \
continuum_imaging_list_arlexecute_workflow(self.vis_list, model_imagelist=self.model_imagelist, context='2d',
algorithm='mmclean', facets=1,
scales=[0, 3, 10],
niter=1000, fractional_threshold=0.1,
nmoments=2, nchan=self.freqwin,
threshold=2.0, nmajor=5, gain=0.1,
deconvolve_facets=8, deconvolve_overlap=16,
deconvolve_taper='tukey')
clean, residual, restored = arlexecute.compute(continuum_imaging_list,
sync=True)
export_image_to_fits(
clean[0],
'%s/test_pipelines_continuum_imaging_pipeline_clean.fits' %
self.dir)
export_image_to_fits(
residual[0][0],
'%s/test_pipelines_continuum_imaging_pipeline_residual.fits' %
self.dir)
export_image_to_fits(
restored[0],
'%s/test_pipelines_continuum_imaging_pipeline_restored.fits' %
self.dir)
qa = qa_image(restored[0])
assert numpy.abs(qa.data['max'] - 116.9) < 1.0, str(qa)
assert numpy.abs(qa.data['min'] + 0.118) < 1.0, str(qa)
def test_continuum_imaging_pipeline_serialclean(self):
self.actualSetUp(add_errors=False, zerow=True)
continuum_imaging_list = \
continuum_imaging_list_arlexecute_workflow(self.vis_list,
model_imagelist=self.model_imagelist,
context='2d',
algorithm='mmclean', facets=1,
scales=[0, 3, 10],
niter=1000, fractional_threshold=0.1, threshold=0.1,
nmoment=3,
nmajor=5, gain=0.1,
deconvolve_facets=4, deconvolve_overlap=32,
deconvolve_taper='tukey', psf_support=64,
use_serial_clean=True)
clean, residual, restored = arlexecute.compute(continuum_imaging_list,
sync=True)
centre = len(clean) // 2
if self.persist:
export_image_to_fits(
clean[centre],
'%s/test_pipelines_continuum_imaging_pipeline_arlexecute_clean.fits'
% self.dir)
export_image_to_fits(
residual[centre][0],
'%s/test_pipelines_continuum_imaging_pipeline_arlexecute_residual.fits'
% self.dir)
export_image_to_fits(
restored[centre],
'%s/test_pipelines_continuum_imaging_pipeline_arlexecute_restored.fits'
% self.dir)
qa = qa_image(restored[centre])
assert numpy.abs(qa.data['max'] - 100.13762476849081) < 1.0, str(qa)
assert numpy.abs(qa.data['min'] + 0.03627273884170454) < 1.0, str(qa)
def _invert_base(self,
context,
extra='',
fluxthreshold=1.0,
positionthreshold=1.0,
check_components=True,
facets=1,
vis_slices=1,
gcfcf=None,
**kwargs):
centre = self.freqwin // 2
dirty = invert_list_arlexecute_workflow(self.vis_list,
self.model_list,
context=context,
dopsf=False,
normalize=True,
facets=facets,
vis_slices=vis_slices,
gcfcf=gcfcf,
**kwargs)
dirty = arlexecute.compute(dirty, sync=True)[centre]
export_image_to_fits(
dirty[0], '%s/test_imaging_invert_%s%s_%s_dirty.fits' %
(self.dir, context, extra, arlexecute.type()))
assert numpy.max(numpy.abs(dirty[0].data)), "Image is empty"
if check_components:
self._checkcomponents(dirty[0], fluxthreshold, positionthreshold)
def _predict_base(self, context='2d', extra='', fluxthreshold=1.0, facets=1, vis_slices=1, **kwargs):
vis_list = zero_list_arlexecute_workflow(self.vis_list)
vis_list = predict_list_arlexecute_workflow(vis_list, self.model_list, context=context,
vis_slices=vis_slices, facets=facets, **kwargs)
vis_list = subtract_list_arlexecute_workflow(self.vis_list, vis_list)[0]
vis_list = arlexecute.compute(vis_list, sync=True)
dirty = invert_list_arlexecute_workflow([vis_list], [self.model_list[0]], context='2d', dopsf=False,
normalize=True)[0]
dirty = arlexecute.compute(dirty, sync=True)
assert numpy.max(numpy.abs(dirty[0].data)), "Residual image is empty"
export_image_to_fits(dirty[0], '%s/test_imaging_predict_%s%s_%s_dirty.fits' %
(self.dir, context, extra, arlexecute.type()))
maxabs = numpy.max(numpy.abs(dirty[0].data))
assert maxabs < fluxthreshold, "Error %.3f greater than fluxthreshold %.3f " % (maxabs, fluxthreshold)
def test_useFunction(self):
def square(x):
return x**2
arlexecute.set_client(use_dask=False)
graph = arlexecute.execute(square)(numpy.arange(10))
assert (arlexecute.compute(graph) == numpy.array(
[0, 1, 4, 9, 16, 25, 36, 49, 64, 81])).all()
arlexecute.close()
def test_residual_list(self):
self.actualSetUp(zerow=True)
centre = self.freqwin // 2
residual_image_list = residual_list_arlexecute_workflow(self.vis_list, self.model_list, context='2d')
residual_image_list = arlexecute.compute(residual_image_list, sync=True)
qa = qa_image(residual_image_list[centre][0])
assert numpy.abs(qa.data['max'] - 0.35139716991480785) < 1.0, str(qa)
assert numpy.abs(qa.data['min'] + 0.7681701460717593) < 1.0, str(qa)
def test_useDaskSync(self):
def square(x):
return x**2
arlexecute.set_client(use_dask=True)
graph = arlexecute.execute(square)(numpy.arange(10))
result = arlexecute.compute(graph, sync=True)
assert (result == numpy.array([0, 1, 4, 9, 16, 25, 36, 49, 64,
81])).all()
arlexecute.close()
def test_calibrate_arlexecute_empty(self):
amp_errors = {'T': 0.0, 'G': 0.0}
phase_errors = {'T': 1.0, 'G': 0.0}
self.actualSetUp(amp_errors=amp_errors, phase_errors=phase_errors)
for v in self.blockvis_list:
v.data['vis'][...] = 0.0 + 0.0j
controls = create_calibration_controls()
controls['T']['first_selfcal'] = 0
controls['T']['timescale'] = 'auto'
calibrate_list = \
calibrate_list_arlexecute_workflow(self.error_blockvis_list, self.blockvis_list,
calibration_context='T', controls=controls, do_selfcal=True,
global_solution=False)
calibrate_list = arlexecute.compute(calibrate_list, sync=True)
assert len(calibrate_list[1][0]) > 0
def test_deconvolve_and_restore_cube_mmclean_facets(self):
self.actualSetUp(add_errors=True)
dirty_imagelist = invert_list_arlexecute_workflow(self.vis_list,
self.model_imagelist,
context='2d',
dopsf=False,
normalize=True)
psf_imagelist = invert_list_arlexecute_workflow(self.vis_list,
self.model_imagelist,
context='2d',
dopsf=True,
normalize=True)
dirty_imagelist = arlexecute.persist(dirty_imagelist)
psf_imagelist = arlexecute.persist(psf_imagelist)
dec_imagelist = deconvolve_list_arlexecute_workflow(
dirty_imagelist,
psf_imagelist,
self.model_imagelist,
niter=1000,
fractional_threshold=0.1,
scales=[0, 3, 10],
algorithm='mmclean',
nmoment=3,
nchan=self.freqwin,
threshold=0.01,
gain=0.7,
deconvolve_facets=8,
deconvolve_overlap=8,
deconvolve_taper='tukey')
dec_imagelist = arlexecute.persist(dec_imagelist)
residual_imagelist = residual_list_arlexecute_workflow(
self.vis_list, model_imagelist=dec_imagelist, context='2d')
residual_imagelist = arlexecute.persist(residual_imagelist)
restored_list = restore_list_arlexecute_workflow(
model_imagelist=dec_imagelist,
psf_imagelist=psf_imagelist,
residual_imagelist=residual_imagelist,
empty=self.model_imagelist)
restored = arlexecute.compute(restored_list, sync=True)[0]
export_image_to_fits(
restored, '%s/test_imaging_%s_overlap_mmclean_restored.fits' %
(self.dir, arlexecute.type()))
def test_ical_pipeline(self):
amp_errors = {'T': 0.0, 'G': 0.00, 'B': 0.0}
phase_errors = {'T': 0.1, 'G': 0.0, 'B': 0.0}
self.actualSetUp(add_errors=True,
block=True,
amp_errors=amp_errors,
phase_errors=phase_errors)
controls = create_calibration_controls()
controls['T']['first_selfcal'] = 1
controls['G']['first_selfcal'] = 3
controls['B']['first_selfcal'] = 4
controls['T']['timescale'] = 'auto'
controls['G']['timescale'] = 'auto'
controls['B']['timescale'] = 1e5
ical_list = \
ical_list_arlexecute_workflow(self.vis_list, model_imagelist=self.model_imagelist, context='2d',
calibration_context='T', controls=controls, do_selfcal=True,
global_solution=False,
algorithm='mmclean',
facets=1,
scales=[0, 3, 10],
niter=1000, fractional_threshold=0.1,
nmoments=2, nchan=self.freqwin,
threshold=2.0, nmajor=5, gain=0.1,
deconvolve_facets=8, deconvolve_overlap=16, deconvolve_taper='tukey')
clean, residual, restored = arlexecute.compute(ical_list, sync=True)
centre = len(clean) // 2
export_image_to_fits(
clean[centre],
'%s/test_pipelines_ical_pipeline_clean.fits' % self.dir)
export_image_to_fits(
residual[centre][0],
'%s/test_pipelines_ical_pipeline_residual.fits' % self.dir)
export_image_to_fits(
restored[centre],
'%s/test_pipelines_ical_pipeline_restored.fits' % self.dir)
qa = qa_image(restored[centre])
assert numpy.abs(qa.data['max'] - 100.13739440876233) < 1.0, str(qa)
assert numpy.abs(qa.data['min'] + 0.03644435471804354) < 1.0, str(qa)
def test_deconvolve_and_restore_cube_mmclean(self):
self.actualSetUp(add_errors=True)
dirty_imagelist = invert_list_arlexecute_workflow(self.vis_list, self.model_imagelist, context='2d',
dopsf=False, normalize=True)
psf_imagelist = invert_list_arlexecute_workflow(self.vis_list, self.model_imagelist, context='2d',
dopsf=True, normalize=True)
dec_imagelist, _ = deconvolve_list_arlexecute_workflow(dirty_imagelist, psf_imagelist, self.model_imagelist, niter=1000,
fractional_threshold=0.01, scales=[0, 3, 10],
algorithm='mmclean', nmoments=3, nchan=self.freqwin,
threshold=0.1, gain=0.7)
residual_imagelist = residual_list_arlexecute_workflow(self.vis_list, model_imagelist=dec_imagelist,
context='wstack', vis_slices=51)
restored = restore_list_arlexecute_workflow(model_imagelist=dec_imagelist, psf_imagelist=psf_imagelist,
residual_imagelist=residual_imagelist,
empty=self.model_imagelist)[0]
restored = arlexecute.compute(restored, sync=True)
export_image_to_fits(restored, '%s/test_imaging_%s_mmclean_restored.fits' % (self.dir, arlexecute.type()))
def test_ical_pipeline_global(self):
self.actualSetUp(add_errors=True)
controls = create_calibration_controls()
controls['T']['first_selfcal'] = 1
controls['T']['timescale'] = 'auto'
ical_list = \
ical_list_arlexecute_workflow(self.vis_list,
model_imagelist=self.model_imagelist,
context='2d',
algorithm='mmclean', facets=1,
scales=[0, 3, 10],
niter=1000, fractional_threshold=0.1, threshold=0.1,
nmoment=3,
nmajor=1, gain=0.1,
deconvolve_facets=4, deconvolve_overlap=32,
deconvolve_taper='tukey', psf_support=64,
calibration_context='T', controls=controls, do_selfcal=True,
global_solution=True)
clean, residual, restored, gt_list = arlexecute.compute(ical_list,
sync=True)
centre = len(clean) // 2
if self.persist:
export_image_to_fits(
clean[centre],
'%s/test_pipelines_ical_global_pipeline_arlexecute_clean.fits'
% self.dir)
export_image_to_fits(
residual[centre][0],
'%s/test_pipelines_ical_global_pipeline_arlexecute_residual.fits'
% self.dir)
export_image_to_fits(
restored[centre],
'%s/test_pipelines_ical_global_pipeline_arlexecute_restored.fits'
% self.dir)
export_gaintable_to_hdf5(
gt_list[0]['T'],
'%s/test_pipelines_ical_global_pipeline_arlexecute_gaintable.hdf5'
% self.dir)
qa = qa_image(restored[centre])
assert numpy.abs(qa.data['max'] - 98.92656340122159) < 1.0, str(qa)
assert numpy.abs(qa.data['min'] + 0.7024492707920869) < 1.0, str(qa)
def test_calibrate_arlexecute_global(self):
amp_errors = {'T': 0.0, 'G': 0.0}
phase_errors = {'T': 1.0, 'G': 0.0}
self.actualSetUp(amp_errors=amp_errors, phase_errors=phase_errors)
controls = create_calibration_controls()
controls['T']['first_selfcal'] = 0
controls['T']['timescale'] = 'auto'
calibrate_list = \
calibrate_list_arlexecute_workflow(self.error_blockvis_list, self.blockvis_list,
calibration_context='T', controls=controls, do_selfcal=True,
global_solution=True)
calibrate_list = arlexecute.compute(calibrate_list, sync=True)
assert numpy.max(calibrate_list[1]['T'].residual) < 7e-6, numpy.max(
calibrate_list[1]['T'].residual)
assert numpy.max(
numpy.abs(calibrate_list[0][0].vis -
self.blockvis_list[0].vis)) < 2e-6
def test_predictcal(self):
self.actualSetUp(zerow=True)
future_vis = arlexecute.scatter(self.vis_list[0])
future_skymodel = arlexecute.scatter(self.skymodel_list)
skymodel_vislist = predict_skymodel_list_arlexecute_workflow(future_vis, future_skymodel,
context='2d', docal=True)
skymodel_vislist = arlexecute.compute(skymodel_vislist, sync=True)
vobs = sum_predict_results(skymodel_vislist)
if self.plot:
def plotvis(i, v):
import matplotlib.pyplot as plt
uvr = numpy.hypot(v.u, v.v)
amp = numpy.abs(v.vis[:, 0])
plt.plot(uvr, amp, '.')
plt.title(str(i))
plt.show()
plotvis(0, vobs)
def actualSetUp(self,
nfreqwin=3,
dospectral=True,
dopol=False,
amp_errors=None,
phase_errors=None,
zerow=True):
if amp_errors is None:
amp_errors = {'T': 0.0, 'G': 0.1}
if phase_errors is None:
phase_errors = {'T': 1.0, 'G': 0.0}
self.npixel = 512
self.low = create_named_configuration('LOWBD2', rmax=750.0)
self.freqwin = nfreqwin
self.vis_list = list()
self.ntimes = 1
self.times = numpy.linspace(-3.0, +3.0, self.ntimes) * numpy.pi / 12.0
self.frequency = numpy.linspace(0.8e8, 1.2e8, self.freqwin)
if self.freqwin > 1:
self.channelwidth = numpy.array(
self.freqwin * [self.frequency[1] - self.frequency[0]])
else:
self.channelwidth = numpy.array([1e6])
if dopol:
self.vis_pol = PolarisationFrame('linear')
self.image_pol = PolarisationFrame('stokesIQUV')
f = numpy.array([100.0, 20.0, -10.0, 1.0])
else:
self.vis_pol = PolarisationFrame('stokesI')
self.image_pol = PolarisationFrame('stokesI')
f = numpy.array([100.0])
if dospectral:
flux = numpy.array(
[f * numpy.power(freq / 1e8, -0.7) for freq in self.frequency])
else:
flux = numpy.array([f])
self.phasecentre = SkyCoord(ra=+180.0 * u.deg,
dec=-60.0 * u.deg,
frame='icrs',
equinox='J2000')
self.blockvis_list = [
arlexecute.execute(ingest_unittest_visibility,
nout=1)(self.low, [self.frequency[i]],
[self.channelwidth[i]],
self.times,
self.vis_pol,
self.phasecentre,
block=True,
zerow=zerow) for i in range(nfreqwin)
]
self.blockvis_list = arlexecute.compute(self.blockvis_list, sync=True)
for v in self.blockvis_list:
v.data['vis'][...] = 1.0 + 0.0j
self.error_blockvis_list = [
arlexecute.execute(copy_visibility(v)) for v in self.blockvis_list
]
gt = arlexecute.execute(create_gaintable_from_blockvisibility)(
self.blockvis_list[0])
gt = arlexecute.execute(simulate_gaintable)(gt,
phase_error=0.1,
amplitude_error=0.0,
smooth_channels=1,
leakage=0.0,
seed=180555)
self.error_blockvis_list = [
arlexecute.execute(apply_gaintable)(self.error_blockvis_list[i],
gt)
for i in range(self.freqwin)
]
self.error_blockvis_list = arlexecute.compute(self.error_blockvis_list,
sync=True)
assert numpy.max(
numpy.abs(self.error_blockvis_list[0].vis -
self.blockvis_list[0].vis)) > 0.0
def create_surface_errors_gaintable_arlexecute_workflow(
band,
sub_bvis_list,
sub_components,
vp_directory,
use_radec=False,
elevation_sampling=5.0,
show=False,
basename=''):
def get_band_vp(band, el):
if band == 'B1':
vpa = import_image_from_fits(
'%s/B1_%d_0565_real_interpolated.fits' %
(vp_directory, int(el)))
vpa_imag = import_image_from_fits(
'%s/B1_%d_0565_imag_interpolated.fits' %
(vp_directory, int(el)))
elif band == 'B2':
vpa = import_image_from_fits(
'%s/B2_%d_1360_real_interpolated.fits' %
(vp_directory, int(el)))
vpa_imag = import_image_from_fits(
'%s/B2_%d_1360_imag_interpolated.fits' %
(vp_directory, int(el)))
elif band == 'Ku':
vpa = import_image_from_fits(
'%s/Ku_%d_11700_real_interpolated.fits' %
(vp_directory, int(el)))
vpa_imag = import_image_from_fits(
'%s/Ku_%d_11700_imag_interpolated.fits' %
(vp_directory, int(el)))
else:
raise ValueError("Unknown band %s" % band)
vpa.data = vpa.data + 1j * vpa_imag.data
return vpa
def find_vp(band, vis):
ha = numpy.pi * numpy.average(vis.time) / 43200.0
dec = vis.phasecentre.dec.rad
latitude = vis.configuration.location.lat.rad
az, el = hadec_to_azel(ha, dec, latitude)
el_deg = el * 180.0 / numpy.pi
el_table = max(
0.0,
min(
90.1,
elevation_sampling *
((el_deg + elevation_sampling / 2.0) // elevation_sampling)))
return get_band_vp(band, el_table)
def find_vp_nominal(band):
el_nominal_deg = 45.0
return get_band_vp(band, el_nominal_deg)
error_pt_list = [
arlexecute.execute(create_pointingtable_from_blockvisibility)(bvis)
for bvis in sub_bvis_list
]
no_error_pt_list = [
arlexecute.execute(create_pointingtable_from_blockvisibility)(bvis)
for bvis in sub_bvis_list
]
vp_nominal_list = [
arlexecute.execute(find_vp_nominal)(band) for bv in sub_bvis_list
]
vp_actual_list = [
arlexecute.execute(find_vp)(band, bv) for bv in sub_bvis_list
]
# Create the gain tables, one per Visibility and per component
no_error_gt_list = [
arlexecute.execute(simulate_gaintable_from_pointingtable)(
bvis,
sub_components,
no_error_pt_list[ibv],
vp_nominal_list[ibv],
use_radec=use_radec) for ibv, bvis in enumerate(sub_bvis_list)
]
error_gt_list = [
arlexecute.execute(simulate_gaintable_from_pointingtable)(
bvis,
sub_components,
error_pt_list[ibv],
vp_actual_list[ibv],
use_radec=use_radec) for ibv, bvis in enumerate(sub_bvis_list)
]
if show:
plot_file = 'gaintable.png'
tmp_gt_list = arlexecute.compute(error_gt_list, sync=True)
plot_gaintable(tmp_gt_list, plot_file=plot_file, title=basename)
return no_error_gt_list, error_gt_list
frequency = numpy.linspace(0.9e8, 1.1e8, nfreqwin)
channel_bandwidth = numpy.array(nfreqwin * [frequency[1] - frequency[0]])
times = numpy.linspace(-numpy.pi / 3.0, numpy.pi / 3.0, ntimes)
phasecentre = SkyCoord(ra=+30.0 * u.deg, dec=-60.0 * u.deg, frame='icrs', equinox='J2000')
vis_list = simulate_list_arlexecute_workflow('LOWBD2',
rmax=rmax,
frequency=frequency,
channel_bandwidth=channel_bandwidth,
times=times,
phasecentre=phasecentre,
order='frequency')
print('%d elements in vis_list' % len(vis_list))
print('About to make visibility')
vis_list = arlexecute.persist(vis_list)
vis_list = arlexecute.compute(vis_list, sync=True)
print(vis_list[0])
advice_low = advise_wide_field(vis_list[0], guard_band_image=8.0, delA=0.02, wprojection_planes=1)
advice_high = advise_wide_field(vis_list[-1], guard_band_image=8.0, delA=0.02,
wprojection_planes=1)
vis_slices = advice_low['vis_slices']
npixel = advice_high['npixels2']
cellsize = min(advice_low['cellsize'], advice_high['cellsize'])
def create_pointing_errors_gaintable_arlexecute_workflow(
sub_bvis_list,
sub_components,
sub_vp_list,
use_radec=False,
pointing_error=0.0,
static_pointing_error=None,
global_pointing_error=None,
time_series='',
time_series_type='',
seed=None,
pointing_directory=None,
show=False,
basename=''):
if global_pointing_error is None:
global_pointing_error = [0.0, 0.0]
# One pointing table per visibility
error_pt_list = [
arlexecute.execute(create_pointingtable_from_blockvisibility)(bvis)
for bvis in sub_bvis_list
]
no_error_pt_list = [
arlexecute.execute(create_pointingtable_from_blockvisibility)(bvis)
for bvis in sub_bvis_list
]
if time_series is '':
error_pt_list = [
arlexecute.execute(simulate_pointingtable)(
pt,
pointing_error=pointing_error,
static_pointing_error=static_pointing_error,
global_pointing_error=global_pointing_error,
seed=seed) for ipt, pt in enumerate(error_pt_list)
]
else:
error_pt_list = [
arlexecute.execute(simulate_pointingtable_from_timeseries)(
pt,
type=time_series,
time_series_type=time_series_type,
pointing_directory=pointing_directory,
seed=seed) for ipt, pt in enumerate(error_pt_list)
]
if show:
tmp_error_pt_list = arlexecute.compute(error_pt_list, sync=True)
if time_series != "":
plot_file = 'pointing_error_%s.png' % (time_series_type)
else:
r2s = 180 * 3600.0 / numpy.pi
plot_file = 'pointing_error_dynamic_%.2f_static_(%.2f,%.2f)_global_(%.2f,%.2f).png' % \
(r2s * pointing_error, r2s * static_pointing_error[0], r2s * static_pointing_error[1],
r2s * global_pointing_error[0], r2s * global_pointing_error[1])
plot_pointingtable(tmp_error_pt_list,
plot_file=plot_file,
title=basename)
# Create the gain tables, one per Visibility and per component
no_error_gt_list = [
arlexecute.execute(simulate_gaintable_from_pointingtable)(
bvis,
sub_components,
no_error_pt_list[ibv],
sub_vp_list[ibv],
use_radec=use_radec) for ibv, bvis in enumerate(sub_bvis_list)
]
error_gt_list = [
arlexecute.execute(simulate_gaintable_from_pointingtable)(
bvis,
sub_components,
error_pt_list[ibv],
sub_vp_list[ibv],
use_radec=use_radec) for ibv, bvis in enumerate(sub_bvis_list)
]
if show:
tmp_gt_list = arlexecute.compute(error_gt_list, sync=True)
if time_series_type != "":
plot_file = 'gaintable_%s.png' % time_series_type
else:
r2s = 180 * 3600.0 / numpy.pi
plot_file = 'gaintable_dynamic_%.2f_static_(%.2f,%.2f)_global_(%.2f,%.2f).png' % \
(r2s * pointing_error, r2s * static_pointing_error[0], r2s * static_pointing_error[1],
r2s * global_pointing_error[0], r2s * global_pointing_error[1])
plot_gaintable(tmp_gt_list,
title="%s: dish 0 amplitude gain, %s" %
(basename, time_series_type),
plot_file=plot_file)
return no_error_gt_list, error_gt_list
def actualSetUp(self, add_errors=False, freqwin=3, block=False, dospectral=True, dopol=False, zerow=False,
makegcfcf=False):
self.npixel = 256
self.low = create_named_configuration('LOWBD2', rmax=750.0)
self.freqwin = freqwin
self.vis_list = list()
self.ntimes = 5
self.cellsize = 0.0005
# Choose the interval so that the maximum change in w is smallish
integration_time = numpy.pi * (24 / (12 * 60))
self.times = numpy.linspace(-integration_time * (self.ntimes // 2), integration_time * (self.ntimes // 2),
self.ntimes)
if freqwin > 1:
self.frequency = numpy.linspace(0.8e8, 1.2e8, self.freqwin)
self.channelwidth = numpy.array(freqwin * [self.frequency[1] - self.frequency[0]])
else:
self.frequency = numpy.array([1.0e8])
self.channelwidth = numpy.array([4e7])
if dopol:
self.vis_pol = PolarisationFrame('linear')
self.image_pol = PolarisationFrame('stokesIQUV')
f = numpy.array([100.0, 20.0, -10.0, 1.0])
else:
self.vis_pol = PolarisationFrame('stokesI')
self.image_pol = PolarisationFrame('stokesI')
f = numpy.array([100.0])
if dospectral:
flux = numpy.array([f * numpy.power(freq / 1e8, -0.7) for freq in self.frequency])
else:
flux = numpy.array([f])
self.phasecentre = SkyCoord(ra=+180.0 * u.deg, dec=-60.0 * u.deg, frame='icrs', equinox='J2000')
self.vis_list = [arlexecute.execute(ingest_unittest_visibility)(self.low,
[self.frequency[freqwin]],
[self.channelwidth[freqwin]],
self.times,
self.vis_pol,
self.phasecentre, block=block,
zerow=zerow)
for freqwin, _ in enumerate(self.frequency)]
self.model_list = [arlexecute.execute(create_unittest_model, nout=freqwin)(self.vis_list[freqwin],
self.image_pol,
cellsize=self.cellsize,
npixel=self.npixel)
for freqwin, _ in enumerate(self.frequency)]
self.components_list = [arlexecute.execute(create_unittest_components)(self.model_list[freqwin],
flux[freqwin, :][numpy.newaxis, :],
single=True)
for freqwin, _ in enumerate(self.frequency)]
self.components_list = arlexecute.compute(self.components_list, sync=True)
self.model_list = [arlexecute.execute(insert_skycomponent, nout=1)(self.model_list[freqwin],
self.components_list[freqwin])
for freqwin, _ in enumerate(self.frequency)]
self.model_list = arlexecute.compute(self.model_list, sync=True)
self.vis_list = [arlexecute.execute(predict_skycomponent_visibility)(self.vis_list[freqwin],
self.components_list[freqwin])
for freqwin, _ in enumerate(self.frequency)]
centre = self.freqwin // 2
# Calculate the model convolved with a Gaussian.
self.model = self.model_list[centre]
self.cmodel = smooth_image(self.model)
export_image_to_fits(self.model, '%s/test_imaging_model.fits' % self.dir)
export_image_to_fits(self.cmodel, '%s/test_imaging_cmodel.fits' % self.dir)
if add_errors and block:
self.vis_list = [arlexecute.execute(insert_unittest_errors)(self.vis_list[i])
for i, _ in enumerate(self.frequency)]
self.components = self.components_list[centre]
if makegcfcf:
self.gcfcf = [create_awterm_convolutionfunction(self.model, nw=61, wstep=16.0,
oversampling=8,
support=64,
use_aaf=True)]
self.gcfcf_clipped = [(self.gcfcf[0][0], apply_bounding_box_convolutionfunction(self.gcfcf[0][1],
fractional_level=1e-3))]
self.gcfcf_joint = [create_awterm_convolutionfunction(self.model, nw=11, wstep=16.0,
oversampling=8,
support=64,
use_aaf=True)]
else:
self.gcfcf = None
self.gcfcf_clipped = None
self.gcfcf_joint = None
def trial_case(results,
seed=180555,
context='wstack',
nworkers=8,
threads_per_worker=1,
memory=8,
processes=True,
order='frequency',
nfreqwin=7,
ntimes=3,
rmax=750.0,
facets=1,
wprojection_planes=1,
use_dask=True,
use_serial_imaging=True,
flux_limit=0.3,
nmajor=5,
dft_threshold=1.0,
use_serial_clean=True,
write_fits=False):
""" Single trial for performance-timings
Simulates visibilities from GLEAM including phase errors
Makes dirty image and PSF
Runs ICAL pipeline
The results are in a dictionary:
'context': input - a string describing concisely the purpose of the test
'time overall', overall execution time (s)
'time predict', time to execute GLEAM prediction graph
'time invert', time to make dirty image
'time invert graph', time to make dirty image graph
'time ICAL graph', time to create ICAL graph
'time ICAL', time to execute ICAL graph
'context', type of imaging e.g. 'wstack'
'nworkers', number of workers to create
'threads_per_worker',
'nnodes', Number of nodes,
'processes', 'order', Ordering of data_models
'nfreqwin', Number of frequency windows in simulation
'ntimes', Number of hour angles in simulation
'rmax', Maximum radius of stations used in simulation (m)
'facets', Number of facets in deconvolution and imaging
'wprojection_planes', Number of wprojection planes
'vis_slices', Number of visibility slices (per Visibbility)
'npixel', Number of pixels in image
'cellsize', Cellsize in radians
'seed', Random number seed
'dirty_max', Maximum in dirty image
'dirty_min', Minimum in dirty image
'restored_max',
'restored_min',
'deconvolved_max',
'deconvolved_min',
'residual_max',
'residual_min',
'git_info', GIT hash (not definitive since local mods are possible)
:param results: Initial state
:param seed: Random number seed (used in gain simulations)
:param context: imaging context
:param context: Type of context: '2d'|'timeslice'|'wstack'
:param nworkers: Number of dask workers to use
:param threads_per_worker: Number of threads per worker
:param processes: Use processes instead of threads 'processes'|'threads'
:param order: See simulate_list_list_arlexecute_workflow_workflowkflow
:param nfreqwin: See simulate_list_list_arlexecute_workflow_workflowkflow
:param ntimes: See simulate_list_list_arlexecute_workflow_workflowkflow
:param rmax: See simulate_list_list_arlexecute_workflow_workflowkflow
:param facets: Number of facets to use
:param wprojection_planes: Number of wprojection planes to use
:param use_dask: Use dask or immediate evaluation
:return: results dictionary
"""
if use_dask:
client = get_dask_Client(threads_per_worker=threads_per_worker,
processes=threads_per_worker == 1,
memory_limit=memory * 1024 * 1024 * 1024,
n_workers=nworkers)
arlexecute.set_client(client)
nodes = findNodes(arlexecute.client)
print("Defined %d workers on %d nodes" % (nworkers, len(nodes)))
print("Workers are: %s" % str(nodes))
else:
arlexecute.set_client(use_dask=use_dask)
results['nnodes'] = 1
def init_logging():
logging.basicConfig(
filename='pipelines_arlexecute_timings.log',
filemode='w',
format='%(asctime)s,%(msecs)d %(name)s %(levelname)s %(message)s',
datefmt='%H:%M:%S',
level=logging.INFO)
init_logging()
log = logging.getLogger()
# Initialise logging on the workers. This appears to only work using the process scheduler.
arlexecute.run(init_logging)
def lprint(*args):
log.info(*args)
print(*args)
lprint("Starting pipelines_arlexecute_timings")
numpy.random.seed(seed)
results['seed'] = seed
start_all = time.time()
results['context'] = context
results['hostname'] = socket.gethostname()
results['git_hash'] = git_hash()
results['epoch'] = time.strftime("%Y-%m-%d %H:%M:%S")
lprint("Context is %s" % context)
results['nworkers'] = nworkers
results['threads_per_worker'] = threads_per_worker
results['processes'] = processes
results['memory'] = memory
results['order'] = order
results['nfreqwin'] = nfreqwin
results['ntimes'] = ntimes
results['rmax'] = rmax
results['facets'] = facets
results['wprojection_planes'] = wprojection_planes
results['dft threshold'] = dft_threshold
results['use_dask'] = use_dask
lprint("At start, configuration is:")
lprint(sort_dict(results))
# Parameters determining scale of simulation.
frequency = numpy.linspace(1.0e8, 1.2e8, nfreqwin)
centre = nfreqwin // 2
if nfreqwin > 1:
channel_bandwidth = numpy.array(nfreqwin *
[frequency[1] - frequency[0]])
else:
channel_bandwidth = numpy.array([1e6])
times = numpy.linspace(-numpy.pi / 4.0, numpy.pi / 4.0, ntimes)
phasecentre = SkyCoord(ra=+0.0 * u.deg,
dec=-40.0 * u.deg,
frame='icrs',
equinox='J2000')
lprint("****** Visibility creation ******")
# Create the empty BlockVisibility's and persist these on the cluster
tmp_bvis_list = simulate_list_arlexecute_workflow(
'LOWBD2',
frequency=frequency,
channel_bandwidth=channel_bandwidth,
times=times,
phasecentre=phasecentre,
order=order,
format='blockvis',
rmax=rmax)
tmp_vis_list = [
arlexecute.execute(convert_blockvisibility_to_visibility)(bv)
for bv in tmp_bvis_list
]
tmp_vis_list = arlexecute.client.compute(tmp_vis_list, sync=True)
vis_list = arlexecute.gather(tmp_vis_list)
import matplotlib.pyplot as plt
plt.clf()
plt.hist(vis_list[0].w, bins=100)
plt.title('Histogram of w samples: rms=%.1f (wavelengths)' %
numpy.std(vis_list[0].w))
plt.xlabel('W (wavelengths)')
plt.show()
plt.clf()
plt.hist(vis_list[0].uvdist, bins=100)
plt.title('Histogram of uvdistance samples')
plt.xlabel('UV Distance (wavelengths)')
plt.show()
arlexecute.client.cancel(tmp_vis_list)
future_vis_list = arlexecute.scatter(vis_list)
# Find the best imaging parameters but don't bring the vis_list back here
print("****** Finding wide field parameters ******")
future_advice = [
arlexecute.execute(advise_wide_field)(
v,
guard_band_image=6.0,
delA=0.1,
facets=facets,
wprojection_planes=wprojection_planes,
oversampling_synthesised_beam=4.0) for v in future_vis_list
]
future_advice = arlexecute.compute(future_advice)
advice = arlexecute.client.gather(future_advice)[-1]
arlexecute.client.cancel(future_advice)
# Deconvolution via sub-images requires 2^n
npixel = advice['npixels2']
results['npixel'] = npixel
cellsize = advice['cellsize']
results['cellsize'] = cellsize
lprint("Image will have %d by %d pixels, cellsize = %.6f rad" %
(npixel, npixel, cellsize))
# Create an empty model image
tmp_model_list = [
arlexecute.execute(create_image)(
npixel=npixel,
cellsize=cellsize,
frequency=[frequency[f]],
channel_bandwidth=[channel_bandwidth[f]],
phasecentre=phasecentre,
polarisation_frame=PolarisationFrame("stokesI"))
for f, freq in enumerate(frequency)
]
model_list = arlexecute.compute(tmp_model_list, sync=True)
future_model_list = arlexecute.scatter(model_list)
lprint("****** Setting up imaging parameters ******")
# Now set up the imaging parameters
template_model = create_image(
npixel=npixel,
cellsize=cellsize,
frequency=[frequency[centre]],
phasecentre=phasecentre,
channel_bandwidth=[channel_bandwidth[centre]],
polarisation_frame=PolarisationFrame("stokesI"))
gcfcf = [create_pswf_convolutionfunction(template_model)]
if context == 'timeslice':
vis_slices = ntimes
lprint("Using timeslice with %d slices" % vis_slices)
elif context == '2d':
vis_slices = 1
elif context == "wprojection":
wstep = advice['wstep']
nw = advice['wprojection_planes']
vis_slices = 1
support = advice['nwpixels']
results['wprojection_planes'] = nw
lprint("****** Starting W projection kernel creation ******")
lprint("Using wprojection with %d planes with wstep %.1f wavelengths" %
(nw, wstep))
lprint("Support of wprojection = %d pixels" % support)
gcfcf = [
create_awterm_convolutionfunction(template_model,
nw=nw,
wstep=wstep,
oversampling=4,
support=support,
use_aaf=True)
]
lprint("Size of W projection gcf, cf = %.2E bytes" % get_size(gcfcf))
else:
context = 'wstack'
vis_slices = advice['vis_slices']
lprint("Using wstack with %d slices" % vis_slices)
gcfcf = arlexecute.scatter(gcfcf, broadcast=True)
results['vis_slices'] = vis_slices
# Make a skymodel from gleam, with bright sources as components and weak sources in an image
lprint("****** Starting GLEAM skymodel creation ******")
future_skymodel_list = [
arlexecute.execute(create_low_test_skymodel_from_gleam)(
npixel=npixel,
cellsize=cellsize,
frequency=[frequency[f]],
phasecentre=phasecentre,
polarisation_frame=PolarisationFrame("stokesI"),
flux_limit=flux_limit,
flux_threshold=dft_threshold,
flux_max=5.0) for f, freq in enumerate(frequency)
]
# We use predict_skymodel so that we can use skycomponents as well as images
lprint("****** Starting GLEAM skymodel prediction ******")
predicted_vis_list = [
predict_skymodel_list_arlexecute_workflow(future_vis_list[f],
[future_skymodel_list[f]],
context=context,
vis_slices=vis_slices,
facets=facets,
gcfcf=gcfcf)[0]
for f, freq in enumerate(frequency)
]
# Corrupt the visibility for the GLEAM model
lprint("****** Visibility corruption ******")
tmp_corrupted_vis_list = corrupt_list_arlexecute_workflow(
predicted_vis_list, phase_error=1.0, seed=seed)
lprint("****** Weighting and tapering ******")
tmp_corrupted_vis_list = weight_list_arlexecute_workflow(
tmp_corrupted_vis_list, future_model_list)
tmp_corrupted_vis_list = taper_list_arlexecute_workflow(
tmp_corrupted_vis_list, 0.003 * 750.0 / rmax)
tmp_corrupted_vis_list = arlexecute.compute(tmp_corrupted_vis_list,
sync=True)
corrupted_vis_list = arlexecute.gather(tmp_corrupted_vis_list)
# arlexecute.client.cancel(predicted_vis_list)
arlexecute.client.cancel(tmp_corrupted_vis_list)
future_corrupted_vis_list = arlexecute.scatter(corrupted_vis_list)
# At this point the only futures are of scatter'ed data so no repeated calculations should be
# incurred.
lprint("****** Starting dirty image calculation ******")
start = time.time()
dirty_list = invert_list_arlexecute_workflow(
future_corrupted_vis_list,
future_model_list,
vis_slices=vis_slices,
context=context,
facets=facets,
use_serial_invert=use_serial_imaging,
gcfcf=gcfcf)
results['size invert graph'] = get_size(dirty_list)
lprint('Size of dirty graph is %.3E bytes' %
(results['size invert graph']))
end = time.time()
results['time invert graph'] = end - start
lprint("Construction of invert graph took %.3f seconds" % (end - start))
start = time.time()
dirty, sumwt = arlexecute.compute(dirty_list, sync=True)[centre]
end = time.time()
results['time invert'] = end - start
lprint("Dirty image invert took %.3f seconds" % (end - start))
lprint("Maximum in dirty image is %f, sumwt is %s" %
(numpy.max(numpy.abs(dirty.data)), str(sumwt)))
qa = qa_image(dirty)
results['dirty_max'] = qa.data['max']
results['dirty_min'] = qa.data['min']
if write_fits:
export_image_to_fits(
dirty, "pipelines_arlexecute_timings-%s-dirty.fits" % context)
lprint("****** Starting prediction ******")
start = time.time()
tmp_vis_list = predict_list_arlexecute_workflow(
future_corrupted_vis_list,
future_model_list,
vis_slices=vis_slices,
context=context,
facets=facets,
use_serial_predict=use_serial_imaging,
gcfcf=gcfcf)
result = arlexecute.compute(tmp_vis_list, sync=True)
# arlexecute.client.cancel(tmp_vis_list)
end = time.time()
results['time predict'] = end - start
lprint("Predict took %.3f seconds" % (end - start))
# Create the ICAL pipeline to run major cycles, starting selfcal at cycle 1. A global solution across all
# frequencies (i.e. Visibilities) is performed.
print("Using subimage clean")
deconvolve_facets = 8
deconvolve_overlap = 16
deconvolve_taper = 'tukey'
lprint("****** Starting ICAL graph creation ******")
controls = create_calibration_controls()
controls['T']['first_selfcal'] = 1
controls['T']['timescale'] = 'auto'
start = time.time()
ical_list = ical_list_arlexecute_workflow(
future_corrupted_vis_list,
model_imagelist=future_model_list,
context=context,
vis_slices=vis_slices,
scales=[0, 3, 10],
algorithm='mmclean',
nmoment=3,
niter=1000,
fractional_threshold=0.1,
threshold=0.01,
nmajor=nmajor,
gain=0.25,
psf_support=64,
deconvolve_facets=deconvolve_facets,
deconvolve_overlap=deconvolve_overlap,
deconvolve_taper=deconvolve_taper,
timeslice='auto',
global_solution=True,
do_selfcal=True,
calibration_context='T',
controls=controls,
use_serial_predict=use_serial_imaging,
use_serial_invert=use_serial_imaging,
use_serial_clean=use_serial_clean,
gcfcf=gcfcf)
results['size ICAL graph'] = get_size(ical_list)
lprint('Size of ICAL graph is %.3E bytes' % results['size ICAL graph'])
end = time.time()
results['time ICAL graph'] = end - start
lprint("Construction of ICAL graph took %.3f seconds" % (end - start))
print("Current objects on cluster: ")
pp.pprint(arlexecute.client.who_has())
#
# Execute the graph
lprint("****** Executing ICAL graph ******")
start = time.time()
deconvolved, residual, restored, gaintables = arlexecute.compute(ical_list,
sync=True)
end = time.time()
results['time ICAL'] = end - start
lprint("ICAL graph execution took %.3f seconds" % (end - start))
qa = qa_image(deconvolved[centre])
results['deconvolved_max'] = qa.data['max']
results['deconvolved_min'] = qa.data['min']
deconvolved_cube = image_gather_channels(deconvolved)
if write_fits:
export_image_to_fits(
deconvolved_cube,
"pipelines_arlexecute_timings-%s-ical_deconvolved.fits" % context)
qa = qa_image(residual[centre][0])
results['residual_max'] = qa.data['max']
results['residual_min'] = qa.data['min']
residual_cube = remove_sumwt(residual)
residual_cube = image_gather_channels(residual_cube)
if write_fits:
export_image_to_fits(
residual_cube,
"pipelines_arlexecute_timings-%s-ical_residual.fits" % context)
qa = qa_image(restored[centre])
results['restored_max'] = qa.data['max']
results['restored_min'] = qa.data['min']
restored_cube = image_gather_channels(restored)
if write_fits:
export_image_to_fits(
restored_cube,
"pipelines_arlexecute_timings-%s-ical_restored.fits" % context)
#
arlexecute.close()
end_all = time.time()
results['time overall'] = end_all - start_all
lprint("At end, results are:")
results = sort_dict(results)
lprint(results)
return results
def actualSetUp(self,
add_errors=False,
freqwin=7,
block=False,
dospectral=True,
dopol=False,
zerow=True):
self.npixel = 256
self.low = create_named_configuration('LOWBD2', rmax=750.0)
self.freqwin = freqwin
self.vis_list = list()
self.ntimes = 5
cellsize = 0.001
self.times = numpy.linspace(-3.0, +3.0, self.ntimes) * numpy.pi / 12.0
self.frequency = numpy.linspace(0.8e8, 1.2e8, self.freqwin)
if freqwin > 1:
self.channelwidth = numpy.array(
freqwin * [self.frequency[1] - self.frequency[0]])
else:
self.channelwidth = numpy.array([1e6])
if dopol:
self.vis_pol = PolarisationFrame('linear')
self.image_pol = PolarisationFrame('stokesIQUV')
f = numpy.array([100.0, 20.0, -10.0, 1.0])
else:
self.vis_pol = PolarisationFrame('stokesI')
self.image_pol = PolarisationFrame('stokesI')
f = numpy.array([100.0])
if dospectral:
flux = numpy.array(
[f * numpy.power(freq / 1e8, -0.7) for freq in self.frequency])
else:
flux = numpy.array([f])
self.phasecentre = SkyCoord(ra=+180.0 * u.deg,
dec=-60.0 * u.deg,
frame='icrs',
equinox='J2000')
self.vis_list = [
arlexecute.execute(ingest_unittest_visibility)(
self.low, [self.frequency[freqwin]],
[self.channelwidth[freqwin]],
self.times,
self.vis_pol,
self.phasecentre,
block=block,
zerow=zerow) for freqwin, _ in enumerate(self.frequency)
]
self.model_imagelist = [
arlexecute.execute(create_unittest_model,
nout=freqwin)(self.vis_list[freqwin],
self.image_pol,
cellsize=cellsize,
npixel=self.npixel)
for freqwin, _ in enumerate(self.frequency)
]
self.componentlist = [
arlexecute.execute(create_unittest_components)(
self.model_imagelist[freqwin],
flux[freqwin, :][numpy.newaxis, :])
for freqwin, _ in enumerate(self.frequency)
]
self.model_imagelist = [
arlexecute.execute(insert_skycomponent,
nout=1)(self.model_imagelist[freqwin],
self.componentlist[freqwin])
for freqwin, _ in enumerate(self.frequency)
]
self.vis_list = [
arlexecute.execute(predict_skycomponent_visibility)(
self.vis_list[freqwin], self.componentlist[freqwin])
for freqwin, _ in enumerate(self.frequency)
]
# Calculate the model convolved with a Gaussian.
self.model_imagelist = arlexecute.compute(self.model_imagelist,
sync=True)
model = self.model_imagelist[0]
self.cmodel = smooth_image(model)
export_image_to_fits(
model,
'%s/test_imaging_arlexecute_deconvolved_model.fits' % self.dir)
export_image_to_fits(
self.cmodel,
'%s/test_imaging_arlexecute_deconvolved_cmodel.fits' % self.dir)
if add_errors and block:
self.vis_list = [
arlexecute.execute(insert_unittest_errors)(self.vis_list[i])
for i, _ in enumerate(self.frequency)
]
# self.vis_list = arlexecute.compute(self.vis_list, sync=True)
self.vis_list = arlexecute.persist(self.vis_list)
self.model_imagelist = arlexecute.scatter(self.model_imagelist)
threshold=0.1,
nmajor=5,
gain=0.25,
deconvolve_facets=8,
deconvolve_overlap=32,
deconvolve_taper='tukey',
vis_slices=ntimes,
timeslice='auto',
global_solution=False,
psf_support=64,
do_selfcal=True)
# In[ ]:
log.info('About to run ical_list_serial_workflow')
result = arlexecute.compute(ical_list, sync=True)
deconvolved = result[0][0]
residual = result[1][0]
restored = result[2][0]
arlexecute.close()
show_image(deconvolved,
title='Clean image',
cm='Greys',
vmax=0.1,
vmin=-0.01)
print(qa_image(deconvolved, context='Clean image'))
plt.show()
export_image_to_fits(
deconvolved,
'%s/gleam_ical_arlexecute_deconvolved.fits' % (results_dir))
print(contexts)
#dict_keys(['facets_wstack', 'facets_timeslice', '2d', 'wstack', 'wprojection', 'timeslice', 'facets', 'wsnapshots'])
print(gcfcf_clipped[1])
contexts = ['2d', 'facets', 'timeslice', 'wstack', 'wprojection']
for context in contexts:
print('Processing context %s' % context)
vtpredict_list =[create_visibility(lowcore, times, frequency, channel_bandwidth=channel_bandwidth,
weight=1.0, phasecentre=phasecentre, polarisation_frame=PolarisationFrame('stokesI'))]
model_list = [model]
vtpredict_list = arlexecute.compute(vtpredict_list, sync=True)
vtpredict_list = arlexecute.scatter(vtpredict_list)
if context == 'wprojection':
future = predict_list_arlexecute_workflow(vtpredict_list, model_list, context='2d', gcfcf=[gcfcf_clipped])
elif context == 'facets':
future = predict_list_arlexecute_workflow(vtpredict_list, model_list, context=context, facets=8)
elif context == 'timeslice':
future = predict_list_arlexecute_workflow(vtpredict_list, model_list, context=context, vis_slices=vis_timeslices(
vtpredict, 'auto'))
elif context == 'wstack':
future = predict_list_arlexecute_workflow(vtpredict_list, model_list, context=context, vis_slices=31)
def actualSetUp(self,
add_errors=False,
nfreqwin=7,
dospectral=True,
dopol=False,
zerow=True):
self.npixel = 512
self.low = create_named_configuration('LOWBD2', rmax=750.0)
self.freqwin = nfreqwin
self.vis_list = list()
self.ntimes = 5
self.times = numpy.linspace(-3.0, +3.0, self.ntimes) * numpy.pi / 12.0
self.frequency = numpy.linspace(0.8e8, 1.2e8, self.freqwin)
if self.freqwin > 1:
self.channelwidth = numpy.array(
self.freqwin * [self.frequency[1] - self.frequency[0]])
else:
self.channelwidth = numpy.array([1e6])
if dopol:
self.vis_pol = PolarisationFrame('linear')
self.image_pol = PolarisationFrame('stokesIQUV')
f = numpy.array([100.0, 20.0, -10.0, 1.0])
else:
self.vis_pol = PolarisationFrame('stokesI')
self.image_pol = PolarisationFrame('stokesI')
f = numpy.array([100.0])
if dospectral:
flux = numpy.array(
[f * numpy.power(freq / 1e8, -0.7) for freq in self.frequency])
else:
flux = numpy.array([f])
self.phasecentre = SkyCoord(ra=+180.0 * u.deg,
dec=-60.0 * u.deg,
frame='icrs',
equinox='J2000')
self.blockvis_list = [
arlexecute.execute(ingest_unittest_visibility,
nout=1)(self.low, [self.frequency[i]],
[self.channelwidth[i]],
self.times,
self.vis_pol,
self.phasecentre,
block=True,
zerow=zerow) for i in range(nfreqwin)
]
self.blockvis_list = arlexecute.compute(self.blockvis_list, sync=True)
self.blockvis_list = arlexecute.scatter(self.blockvis_list)
self.vis_list = [
arlexecute.execute(convert_blockvisibility_to_visibility,
nout=1)(bv) for bv in self.blockvis_list
]
self.vis_list = arlexecute.compute(self.vis_list, sync=True)
self.vis_list = arlexecute.scatter(self.vis_list)
self.model_imagelist = [
arlexecute.execute(create_unittest_model,
nout=1)(self.vis_list[i],
self.image_pol,
npixel=self.npixel,
cellsize=0.0005)
for i in range(nfreqwin)
]
self.model_imagelist = arlexecute.compute(self.model_imagelist,
sync=True)
self.model_imagelist = arlexecute.scatter(self.model_imagelist)
self.components_list = [
arlexecute.execute(create_unittest_components)(
self.model_imagelist[freqwin],
flux[freqwin, :][numpy.newaxis, :])
for freqwin, m in enumerate(self.model_imagelist)
]
self.components_list = arlexecute.compute(self.components_list,
sync=True)
self.components_list = arlexecute.scatter(self.components_list)
self.blockvis_list = [
arlexecute.execute(predict_skycomponent_visibility)(
self.blockvis_list[freqwin], self.components_list[freqwin])
for freqwin, _ in enumerate(self.blockvis_list)
]
self.blockvis_list = arlexecute.compute(self.blockvis_list, sync=True)
self.vis = self.blockvis_list[0]
self.blockvis_list = arlexecute.scatter(self.blockvis_list)
self.model_imagelist = [
arlexecute.execute(insert_skycomponent,
nout=1)(self.model_imagelist[freqwin],
self.components_list[freqwin])
for freqwin in range(nfreqwin)
]
self.model_imagelist = arlexecute.compute(self.model_imagelist,
sync=True)
model = self.model_imagelist[0]
self.cmodel = smooth_image(model)
if self.persist:
export_image_to_fits(
model, '%s/test_pipelines_arlexecute_model.fits' % self.dir)
export_image_to_fits(
self.cmodel,
'%s/test_pipelines_arlexecute_cmodel.fits' % self.dir)
if add_errors:
gt = create_gaintable_from_blockvisibility(self.vis)
gt = simulate_gaintable(gt,
phase_error=0.1,
amplitude_error=0.0,
smooth_channels=1,
leakage=0.0,
seed=180555)
self.blockvis_list = [
arlexecute.execute(apply_gaintable,
nout=1)(self.blockvis_list[i], gt)
for i in range(self.freqwin)
]
self.blockvis_list = arlexecute.compute(self.blockvis_list,
sync=True)
self.blockvis_list = arlexecute.scatter(self.blockvis_list)
self.vis_list = [
arlexecute.execute(convert_blockvisibility_to_visibility)(bv)
for bv in self.blockvis_list
]
self.vis_list = arlexecute.compute(self.vis_list, sync=True)
self.vis_list = arlexecute.scatter(self.vis_list)
self.model_imagelist = [
arlexecute.execute(create_unittest_model,
nout=1)(self.vis_list[i],
self.image_pol,
npixel=self.npixel,
cellsize=0.0005)
for i in range(nfreqwin)
]
self.model_imagelist = arlexecute.compute(self.model_imagelist,
sync=True)
self.model_imagelist = arlexecute.scatter(self.model_imagelist)
