def test_selfcal_nonglobal_graph(self):
corrupted_vis_graph_list = self.setupVis(add_errors=True)
selfcal_vis_graph_list = create_selfcal_graph_list(corrupted_vis_graph_list,
delayed(self.actualmodel),
global_solution=False,
c_predict_graph=create_predict_wstack_graph,
vis_slices=self.vis_slices)
dirty_graph = create_invert_wstack_graph(selfcal_vis_graph_list, self.model_graph,
dopsf=False, normalize=True,
vis_slices=self.vis_slices)
if self.compute:
dirty = dirty_graph.compute()
export_image_to_fits(dirty[0], '%s/test_imaging_graphs_nonglobal_selfcal_dirty.fits' % (self.results_dir))
qa = qa_image(dirty[0])
assert numpy.abs(qa.data['max'] - 101.7) < 1.0, str(qa)
assert numpy.abs(qa.data['min'] + 3.5) < 1.0, str(qa)
def test_deconvolve_msmfsclean_no_taylor(self):
self.comp, self.residual = deconvolve_cube(self.dirty,
self.psf,
niter=self.niter,
gain=0.1,
algorithm='msmfsclean',
scales=[0, 3, 10],
threshold=0.01,
nmoments=1,
findpeak='ARL',
fractional_threshold=0.01,
window=self.innerquarter)
export_image_to_fits(
self.comp,
"%s/test_deconvolve_msmfsclean_notaylor-comp.fits" % (self.dir))
export_image_to_fits(
self.residual,
"%s/test_deconvolve_msmfsclean_notaylor-residual.fits" %
(self.dir))
self.cmodel = restore_cube(self.comp, self.psf, self.residual)
export_image_to_fits(
self.cmodel,
"%s/test_deconvolve_msmfsclean_notaylor-clean.fits" % (self.dir))
assert numpy.max(self.residual.data) < 1.4
def test_ical_pipeline(self):
self.actualSetUp(add_errors=True, block=True)
controls = create_calibration_controls()
controls['T']['first_selfcal'] = 2
controls['G']['first_selfcal'] = 3
controls['B']['first_selfcal'] = 4
controls['T']['timescale'] = 'auto'
controls['G']['timescale'] = 'auto'
controls['B']['timescale'] = 1e5
ical_graph = \
create_ical_pipeline_graph(self.vis_graph_list, model_graph=self.model_graph, context='wstack',
do_selfcal=1, global_solution=False, algorithm='mmclean', vis_slices=51,
facets=1, niter=1000, fractional_threshold=0.1, nmoments=3, nchan=self.freqwin,
threshold=2.0, nmajor=6, gain=0.1)
if self.compute:
clean, residual, restored = ical_graph.compute()
export_image_to_fits(
clean[0],
'%s/test_pipelines_ical_pipeline_clean.fits' % self.dir)
export_image_to_fits(
residual[0][0],
'%s/test_pipelines_ical_pipeline_residual.fits' % self.dir)
export_image_to_fits(
restored[0],
'%s/test_pipelines_ical_pipeline_restored.fits' % self.dir)
qa = qa_image(restored[0])
assert numpy.abs(qa.data['max'] - 116.86978265) < 5.0, str(qa)
assert numpy.abs(qa.data['min'] + 0.323425377573) < 5.0, str(qa)
def ingest_visibility(self, freq=1e8, chan_width=1e6, times=[0.0], reffrequency=[1e8], add_errors=False):
lowcore = create_named_configuration('LOWBD2-CORE')
frequency = numpy.array([freq])
channel_bandwidth = numpy.array([chan_width])
# phasecentre = SkyCoord(ra=+180.0 * u.deg, dec=-60.0 * u.deg, frame='icrs', equinox='J2000')
# Observe at zenith to ensure that timeslicing works well. We test that elsewhere.
phasecentre = SkyCoord(ra=+180.0 * u.deg, dec=-60.0 * u.deg, frame='icrs', equinox='J2000')
vt = create_blockvisibility(lowcore, times, frequency, channel_bandwidth=channel_bandwidth,
weight=1.0, phasecentre=phasecentre,
polarisation_frame=PolarisationFrame("stokesI"))
cellsize = 0.001
model = create_image_from_visibility(vt, npixel=self.npixel, cellsize=cellsize, npol=1,
frequency=reffrequency,
polarisation_frame=PolarisationFrame("stokesI"))
flux = numpy.array([[100.0]])
facets = 4
rpix = model.wcs.wcs.crpix - 1
spacing_pixels = self.npixel // facets
centers = [-1.5, -0.5, 0.5, 1.5]
comps = list()
for iy in centers:
for ix in centers:
p = int(round(rpix[0] + ix * spacing_pixels * numpy.sign(model.wcs.wcs.cdelt[0]))), \
int(round(rpix[1] + iy * spacing_pixels * numpy.sign(model.wcs.wcs.cdelt[1])))
sc = pixel_to_skycoord(p[0], p[1], model.wcs, origin=0)
comps.append(create_skycomponent(flux=flux, frequency=vt.frequency, direction=sc,
polarisation_frame=PolarisationFrame("stokesI")))
predict_skycomponent_blockvisibility(vt, comps)
insert_skycomponent(model, comps)
self.actualmodel = copy_image(model)
export_image_to_fits(model, '%s/test_imaging_model.fits' % (self.results_dir))
if add_errors:
# These will be the same for all calls
numpy.random.seed(180555)
gt = create_gaintable_from_blockvisibility(vt)
gt = simulate_gaintable(gt, phase_error=1.0, amplitude_error=0.0)
vt = apply_gaintable(vt, gt)
return vt
def test_deconvolve_mmclean_quadratic_psf(self):
self.comp, self.residual = deconvolve_cube(self.dirty,
self.psf,
niter=self.niter,
gain=0.1,
algorithm='mmclean',
scales=[0, 3, 10],
threshold=0.01,
nmoments=2,
findpeak='ARL',
fractional_threshold=0.01,
window=self.innerquarter,
psf_support=32)
export_image_to_fits(
self.comp,
"%s/test_deconvolve_mmclean_quadratic_psf-comp.fits" % self.dir)
export_image_to_fits(
self.residual,
"%s/test_deconvolve_mmclean_quadratic_psf-residual.fits" %
self.dir)
self.cmodel = restore_cube(self.comp, self.psf, self.residual)
export_image_to_fits(
self.cmodel,
"%s/test_deconvolve_mmclean_quadratic_psf-clean.fits" % self.dir)
assert numpy.max(self.residual.data) < 3.0
def test_continuum_imaging_pipeline(self):
self.actualSetUp(add_errors=False, block=True)
continuum_imaging_graph = \
create_continuum_imaging_pipeline_graph(self.vis_graph_list, model_graph=self.model_graph,
algorithm='mmclean',
nmoments=3, nchan=self.freqwin,
context='wstack', niter=1000, fractional_threshold=0.1,
threshold=2.0, nmajor=0, gain=0.1, vis_slices=51)
if self.compute:
clean, residual, restored = continuum_imaging_graph.compute()
export_image_to_fits(
clean[0],
'%s/test_pipelines_continuum_imaging_pipeline_clean.fits' %
self.dir)
export_image_to_fits(
residual[0][0],
'%s/test_pipelines_continuum_imaging_pipeline_residual.fits' %
self.dir)
export_image_to_fits(
restored[0],
'%s/test_pipelines_continuum_imaging_pipeline_restored.fits' %
self.dir)
qa = qa_image(restored[0])
assert numpy.abs(qa.data['max'] - 116.86978265) < 5.0, str(qa)
assert numpy.abs(qa.data['min'] + 0.323425377573) < 5.0, str(qa)
def test_continuum_imaging_pipeline(self): \
# Note that the image is poor because we set the number of wstack's to be smaller than
# recommended. Setting it to e.g. 51 gives a better image but at the cost of longer run time.
self.actualSetUp(add_errors=False, block=True)
continuum_imaging_graph = \
create_continuum_imaging_pipeline_graph(self.vis_graph_list, model_graph=self.model_graph,
algorithm='mmclean',
nmoments=3, nchan=self.freqwin,
context='wstack', niter=1000, fractional_threshold=0.1,
threshold=2.0, nmajor=0, gain=0.1, vis_slices=11)
if self.compute:
clean, residual, restored = continuum_imaging_graph.compute()
export_image_to_fits(
clean[0],
'%s/test_pipelines_continuum_imaging_pipeline_clean.fits' %
self.dir)
export_image_to_fits(
residual[0][0],
'%s/test_pipelines_continuum_imaging_pipeline_residual.fits' %
self.dir)
export_image_to_fits(
restored[0],
'%s/test_pipelines_continuum_imaging_pipeline_restored.fits' %
self.dir)
qa = qa_image(restored[0])
assert numpy.abs(qa.data['max'] - 116.86978265) < 5.0, str(qa)
assert numpy.abs(qa.data['min'] + 0.323425377573) < 5.0, str(qa)
def test_ical_pipeline_global(self):
self.setupVis(add_errors=True)
ical_graph = \
create_ical_pipeline_graph(self.vis_graph_list, model_graph=self.model_graph,
c_deconvolve_graph=create_deconvolve_facet_graph,
c_predict_graph=create_predict_wstack_graph,
c_invert_graph=create_invert_wstack_graph,
c_residual_graph=create_residual_wstack_graph,
c_selfcal_graph=create_selfcal_graph_list,
global_solution=True,
vis_slices=self.vis_slices, facets=2,
niter=1000, fractional_threshold=0.1,
threshold=2.0, nmajor=5, gain=0.1, first_selfcal=1)
if self.compute:
clean, residual, restored = ical_graph.compute()
export_image_to_fits(
clean, '%s/test_pipelines_ical_pipeline_global_clean.fits' %
(self.results_dir))
export_image_to_fits(
residual[0],
'%s/test_pipelines_ical_pipeline_global_residual.fits' %
(self.results_dir))
export_image_to_fits(
restored,
'%s/test_pipelines_ical_pipeline_global_restored.fits' %
(self.results_dir))
qa = qa_image(restored)
assert numpy.abs(qa.data['max'] - 100.0) < 5.0, str(qa)
assert numpy.abs(qa.data['min'] + 5.0) < 5.0, str(qa)
def test_ICAL_bandpass(self):
self.setupVis(add_errors=True, block=True, freqwin=32, bandpass=True)
model = create_empty_image_like(self.model)
comp, residual, restored = ical(self.vis,
model,
algorithm='msclean',
context='wstack',
vis_slices=51,
scales=[0, 3, 10, 30],
threshold=0.01,
findpeak='ARL',
fractional_threshold=0.01,
T_first_selfcal=2,
G_first_selfcal=3,
B_first_selfcal=4,
nmajor=5)
export_image_to_fits(
comp,
"%s/test_pipelines-ical-deconvolved-bandpass.fits" % (self.dir))
export_image_to_fits(
residual,
"%s/test_pipelines-ical-residual-bandpass.fits" % (self.dir))
export_image_to_fits(
restored,
"%s/test_pipelines-ical-restored-bandpass.fits" % (self.dir))
def test_continuum_imaging_pipeline(self):
continuum_imaging_graph = \
create_continuum_imaging_pipeline_graph(self.vis_graph_list, model_graph=self.model_graph,
c_deconvolve_graph=create_deconvolve_facet_graph,
c_invert_graph=create_invert_wstack_graph,
c_residual_graph=create_residual_wstack_graph,
vis_slices=self.vis_slices, facets=2,
niter=1000, fractional_threshold=0.1,
threshold=2.0, nmajor=0, gain=0.1)
if self.compute:
clean, residual, restored = continuum_imaging_graph.compute()
export_image_to_fits(
clean,
'%s/test_pipelines_continuum_imaging_pipeline_clean.fits' %
(self.results_dir))
export_image_to_fits(
residual[0],
'%s/test_pipelines_continuum_imaging_pipeline_residual.fits' %
(self.results_dir))
export_image_to_fits(
restored,
'%s/test_pipelines_continuum_imaging_pipeline_restored.fits' %
(self.results_dir))
qa = qa_image(restored)
assert numpy.abs(qa.data['max'] - 100.0) < 5.0, str(qa)
assert numpy.abs(qa.data['min'] + 5.0) < 5.0, str(qa)
def test_invert_bag(self):
peaks = {
'2d': 65.440798589,
'timeslice': 99.7403479215,
'wstack': 100.654001673
}
vis_slices = {'2d': None, 'timeslice': 'auto', 'wstack': 101}
model = copy_image(self.model)
for context in ['wstack', '2d', 'timeslice']:
dirty_bag = invert_bag(self.vis_bag,
model,
dopsf=False,
context=context,
normalize=True,
vis_slices=vis_slices[context])
dirty, sumwt = list(dirty_bag)[0]
export_image_to_fits(
dirty,
'%s/test_bag_%s_dirty.fits' % (self.results_dir, context))
qa = qa_image(dirty, context=context)
assert numpy.abs(qa.data['max'] - peaks[context]) < 1.0e-7, str(qa)
def setUp(self):
self.dir = './test_results'
self.lowcore = create_named_configuration('LOWBD2-CORE')
os.makedirs(self.dir, exist_ok=True)
self.times = (numpy.pi / (12.0)) * numpy.linspace(-3.0, 3.0, 7)
self.frequency = numpy.array([1e8])
self.channel_bandwidth = numpy.array([1e6])
self.phasecentre = SkyCoord(ra=+180.0 * u.deg, dec=-60.0 * u.deg, frame='icrs', equinox='J2000')
self.vis = create_visibility(self.lowcore, self.times, self.frequency,
channel_bandwidth=self.channel_bandwidth,
phasecentre=self.phasecentre, weight=1.0,
polarisation_frame=PolarisationFrame('stokesI'))
self.vis.data['vis'] *= 0.0
# Create model
self.model = create_test_image(cellsize=0.0015, phasecentre=self.vis.phasecentre,
frequency=self.frequency)
self.model.data[self.model.data > 1.0] = 1.0
self.vis = predict_2d(self.vis, self.model)
assert numpy.max(numpy.abs(self.vis.vis)) > 0.0
export_image_to_fits(self.model, '%s/test_solve_skycomponent_model.fits' % (self.dir))
self.bigmodel = create_image_from_visibility(self.vis, cellsize=0.0015, npixel=512)
def test_predict_bag(self):
errors = {'2d': 28.0, 'timeslice': 31.0, 'wstack': 2.3}
vis_slices = {'2d': None, 'timeslice': 'auto', 'wstack': 101}
for context in ['2d', 'timeslice', 'wstack']:
model_vis_bag = predict_bag(self.vis_bag,
self.model,
context,
vis_slices=vis_slices[context])
error_vis_bag = residual_vis_bag(self.vis_bag, model_vis_bag)
error_image_bag = invert_bag(error_vis_bag,
self.model,
dopsf=False,
context=context,
normalize=True,
vis_slices=vis_slices[context])
result = list(error_image_bag)[0]
error_image = result[0]
export_image_to_fits(
error_image, '%s/test_bag_%s_predict_error_image.fits' %
(self.results_dir, context))
qa = qa_image(error_image, context='error image for %s' % context)
assert qa.data['max'] < errors[context], str(qa)
def test_deconvolution_facet_graph(self):
facets = 4
model_graph = delayed(self.get_LSM)(self.vis_graph_list[self.nvis //
2],
npixel=self.npixel,
flux=0.0)
dirty_graph = create_invert_facet_graph(self.vis_graph_list,
model_graph,
vis_slices=self.vis_slices,
dopsf=False)
psf_model_graph = delayed(self.get_LSM)(
self.vis_graph_list[self.nvis // 2], npixel=self.npixel, flux=0.0)
psf_graph = create_invert_wstack_graph(self.vis_graph_list,
psf_model_graph,
vis_slices=self.vis_slices,
dopsf=True)
clean_graph = create_deconvolve_facet_graph(dirty_graph,
psf_graph,
model_graph,
algorithm='hogbom',
niter=1000,
fractional_threshold=0.02,
threshold=2.0,
gain=0.1,
facets=facets)
result = clean_graph.compute()
export_image_to_fits(
result, '%s/test_imaging_dask_deconvolution_facets%d.clean.fits' %
(self.results_dir, facets))
qa = qa_image(result)
assert numpy.abs(qa.data['max'] - 94.0) < 5.0
assert numpy.abs(qa.data['min'] + 8.0) < 5.0
def test_deconvolve_and_restore_cube_mmclean(self):
self.actualSetUp(add_errors=True)
dirty_graph = create_invert_graph(self.vis_graph_list,
self.model_graph,
context='wstack',
vis_slices=51,
dopsf=False,
normalize=True)
psf_graph = create_invert_graph(self.vis_graph_list,
self.model_graph,
context='wstack',
vis_slices=51,
dopsf=True,
normalize=True)
dec_graph = create_deconvolve_graph(dirty_graph,
psf_graph,
self.model_graph,
niter=1000,
fractional_threshold=0.1,
scales=[0, 3, 10],
algorithm='mmclean',
nmoments=3,
nchan=self.freqwin,
threshold=0.1,
nmajor=0,
gain=0.7)
residual_graph = create_residual_graph(self.vis_graph_list,
model_graph=dec_graph,
context='wstack',
vis_slices=51)
rest_graph = create_restore_graph(model_graph=dec_graph,
psf_graph=psf_graph,
residual_graph=residual_graph,
empty=self.model_graph)
restored = rest_graph[0].compute()
export_image_to_fits(
restored,
'%s/test_imaging_delayed_mmclean_restored.fits' % self.dir)
def test_psf_location_2d(self):
self.actualSetUp()
psf2d = create_empty_image_like(self.model)
psf2d, sumwt = invert_2d(self.componentvis,
psf2d,
dopsf=True,
**self.params)
export_image_to_fits(
psf2d,
'%s/test_imaging_functions_invert_psf_location.fits' % self.dir)
nchan, npol, ny, nx = psf2d.shape
assert numpy.abs(psf2d.data[0, 0, ny // 2, nx // 2] - 1.0) < 2e-3
imagecentre = pixel_to_skycoord(nx // 2 + 1.0,
ny // 2 + 1.0,
wcs=psf2d.wcs,
origin=1)
assert imagecentre.separation(self.phasecentre).value < 1e-15, \
"Image phase centre %s not as expected %s" % (imagecentre, self.phasecentre)
def test_selfcal_global_bag(self):
self.setupVis(add_errors=True)
selfcal_vis_bag = selfcal_bag(self.vis_bag,
self.model_bag,
global_solution=True,
context='wstack',
vis_slices=51)
dirty_bag = invert_bag(selfcal_vis_bag,
self.model_bag,
dopsf=False,
normalize=True,
context='wstack',
vis_slices=101)
if self.compute:
dirty, sumwt = dirty_bag.compute()[0]['image']
export_image_to_fits(
dirty,
'%s/test_imaging_bags_global_selfcal_dirty.fits' % (self.dir))
qa = qa_image(dirty)
assert numpy.abs(qa.data['max'] - 101.7) < 1.0, str(qa)
assert numpy.abs(qa.data['min'] + 3.5) < 1.0, str(qa)
def test_deconvolve_msclean_1scale(self):
self.comp, self.residual = deconvolve_cube(self.dirty, self.psf, niter=10000, gain=0.1, algorithm='msclean',
scales=[0], threshold=0.01)
export_image_to_fits(self.comp, "%s/test_deconvolve_msclean_1scale-comp.fits" % (self.dir))
export_image_to_fits(self.residual, "%s/test_deconvolve_msclean_1scale-residual.fits" % (self.dir))
self.cmodel = restore_cube(self.comp, self.psf, self.residual)
export_image_to_fits(self.cmodel, "%s/test_deconvolve_msclean_1scale-clean.fits" % (self.dir))
assert numpy.max(self.residual.data) < 0.7
def test_deconvolve_msclean_inner_quarter(self):
self.comp, self.residual = deconvolve_cube(self.dirty, self.psf, window='quarter', niter=1000, gain=0.7,
algorithm='msclean', scales=[0, 3, 10, 30], threshold=0.01)
export_image_to_fits(self.comp, "%s/test_deconvolve_msclean_innerquarter-comp.fits" % (self.dir))
export_image_to_fits(self.residual, "%s/test_deconvolve_msclean_innerquarter-residual.fits" % (self.dir))
self.cmodel = restore_cube(self.comp, self.psf, self.residual)
export_image_to_fits(self.cmodel, "%s/test_deconvolve_msclean_innerquarter-clean.fits" % (self.dir))
assert numpy.max(self.residual.data) < 0.5
def test_invert_2d(self):
# Test if the 2D invert works with w set to zero
# Set w=0 so that the two-dimensional transform should agree exactly with the model.
# Good check on the grid correction in the vis->image direction
self.actualSetUp()
self.componentvis = create_visibility(
self.lowcore,
self.times,
self.frequency,
channel_bandwidth=self.channel_bandwidth,
phasecentre=self.phasecentre,
weight=1.0,
polarisation_frame=self.vis_pol)
self.componentvis.data['uvw'][:, 2] = 0.0
self.componentvis.data['vis'] *= 0.0
# Predict the visibility using direct evaluation
for comp in self.components:
predict_skycomponent_visibility(self.componentvis, comp)
psf2d = create_empty_image_like(self.model)
psf2d, sumwt = invert_2d(self.componentvis,
psf2d,
dopsf=True,
**self.params)
export_image_to_fits(psf2d, '%s/test_invert_2d_psf.fits' % self.dir)
dirty2d = create_empty_image_like(self.model)
dirty2d, sumwt = invert_2d(self.componentvis, dirty2d, **self.params)
export_image_to_fits(dirty2d,
'%s/test_invert_2d_dirty.fits' % self.dir)
self._checkcomponents(dirty2d,
fluxthreshold=20.0,
positionthreshold=1.0)
def test_calculate_image_frequency_moments(self):
frequency = numpy.linspace(0.9e8, 1.1e8, 9)
cube = create_low_test_image_from_gleam(npixel=512,
cellsize=0.0001,
frequency=frequency)
log.debug(
export_image_to_fits(cube,
fitsfile='%s/test_moments_cube.fits' %
(self.dir)))
original_cube = copy_image(cube)
moment_cube = calculate_image_frequency_moments(cube, nmoments=3)
log.debug(
export_image_to_fits(moment_cube,
fitsfile='%s/test_moments_moment_cube.fits' %
(self.dir)))
reconstructed_cube = calculate_image_from_frequency_moments(
cube, moment_cube)
log.debug(
export_image_to_fits(
reconstructed_cube,
fitsfile='%s/test_moments_reconstructed_cube.fits' %
(self.dir)))
error = numpy.std(reconstructed_cube.data - original_cube.data)
assert error < 0.2
def test_restore_bag(self):
cvis_bag = self.vis_bag.map(coalesce_visibility)
peaks = {'timeslice': 103.55068395, 'wstack': 98.790503433}
vis_slices = {'2d': None, 'timeslice': 'auto', 'wstack': 101}
for context in ['timeslice', 'wstack']:
dirty_bag = invert_bag(self.vis_bag, self.model_bag, dopsf=False,
context=context, normalize=True,
vis_slices=vis_slices[context])
psf_bag = invert_bag(self.vis_bag, self.model_bag, dopsf=True,
context=context, normalize=True,
vis_slices=vis_slices[context])
model_bag = deconvolve_bag(dirty_bag, psf_bag, self.empty_model_bag, niter=1000, gain=0.7, algorithm='msclean',
threshold=0.01, window_shape=None)
residual_bag = residual_image_bag(cvis_bag, model_bag, context=context, vis_slices=vis_slices[context])
residual = list(residual_bag)[0][0]
export_image_to_fits(residual, '%s/test_bag_%s_residual.fits' % (self.results_dir, context))
final_bag = restore_bag(model_bag, psf_bag, residual_bag)
final = list(final_bag)[0]
qa = qa_image(final, context=context)
export_image_to_fits(final, '%s/test_bag_%s_restored.fits' % (self.results_dir, context))
assert numpy.abs(qa.data['max'] - peaks[context]) < 1.0e-2, str(qa)
def _invert_base(self,
context='2d',
extra='',
positionthreshold=1.0,
check_components=True):
dirty_graph = create_invert_graph(self.vis_graph_list,
self.model_graph,
context=context,
dopsf=False,
normalize=True,
**self.params)
if self.compute:
dirty = dirty_graph[0].compute()
export_image_to_fits(
dirty[0], '%s/test_imaging_graph_invert_%s%s_dirty.fits' % (
self.results_dir,
context,
extra,
))
qa = qa_image(dirty[0])
assert numpy.abs(qa.data['max'] - 100.0) < 5.0, str(qa)
assert numpy.abs(qa.data['min'] + 2.0) < 5.0, str(qa)
def test_deconvolve_msclean_subpsf(self):
self.comp, self.residual = deconvolve_cube(self.dirty, psf=self.psf, psf_support=200,
window=self.innerquarter, niter=1000, gain=0.7,
algorithm='msclean', scales=[0, 3, 10, 30], threshold=0.01)
export_image_to_fits(self.comp, "%s/test_deconvolve_msclean_subpsf-comp.fits" % (self.dir))
export_image_to_fits(self.residual, "%s/test_deconvolve_msclean_subpsf-residual.fits" % (self.dir))
self.cmodel = restore_cube(self.comp, self.psf, self.residual)
export_image_to_fits(self.cmodel, "%s/test_deconvolve_msclean_subpsf-clean.fits" % (self.dir))
assert numpy.max(self.residual.data[..., 56:456, 56:456]) < 1.0
def test_create_image_from_array(self):
m31model_by_array = create_image_from_array(self.m31image.data,
wcs=None)
m31model_by_array = create_image_from_array(self.m31image.data,
self.m31image.wcs)
m31modelsum = add_image(self.m31image, m31model_by_array)
m31modelsum = add_image(self.m31image,
m31model_by_array,
docheckwcs=True)
assert m31model_by_array.shape == self.m31image.shape
log.debug(
export_image_to_fits(self.m31image,
fitsfile='%s/test_model.fits' % (self.dir)))
log.debug(qa_image(m31model_by_array,
context='test_create_from_image'))
def test_continuum_imaging_pipeline(self):
continuum_imaging_graph = \
create_continuum_imaging_pipeline_graph(self.vis_graph_list, model_graph=self.model_graph,
context='wstack', niter=1000, fractional_threshold=0.1,
threshold=2.0, nmajor=0, gain=0.1)
if self.compute:
clean, residual, restored = continuum_imaging_graph.compute()
export_image_to_fits(clean[0], '%s/test_pipelines_continuum_imaging_pipeline_clean.fits' % self.dir)
export_image_to_fits(residual[0],
'%s/test_pipelines_continuum_imaging_pipeline_residual.fits' % self.dir)
export_image_to_fits(restored,
'%s/test_pipelines_continuum_imaging_pipeline_restored.fits' % self.dir)
qa = qa_image(restored)
assert numpy.abs(qa.data['max'] - 100.0) < 5.0, str(qa)
assert numpy.abs(qa.data['min'] + 5.0) < 5.0, str(qa)
def test_deconvolve_and_restore_cube_mmclean(self):
self.bigmodel.data *= 0.0
visres, model, residual = solve_image(self.vis,
self.bigmodel,
nmajor=3,
niter=1000,
threshold=0.01,
gain=0.7,
window='quarter',
scales=[0, 3, 10, 30],
fractional_threshold=0.1,
algorithm='mfsmsclean',
nmoments=3)
export_image_to_fits(
model, '%s/test_solve_image_mmclean_solution.fits' % (self.dir))
export_image_to_fits(
residual, '%s/test_solve_image_mmclean_residual.fits' % (self.dir))
psf, sumwt = invert_2d(self.vis, model, dopsf=True)
export_image_to_fits(
psf, '%s/test_solve_image_mmclean_psf.fits' % (self.dir))
restored = restore_cube(model=model, psf=psf, residual=residual)
export_image_to_fits(
restored, '%s/test_solve_image_mmclean_restored.fits' % (self.dir))
assert numpy.max(numpy.abs(residual.data)) < 1.2
def ingest_visibility(self,
freq=[1e8],
chan_width=[1e6],
times=None,
reffrequency=None,
add_errors=False,
block=True):
if times is None:
times = (numpy.pi / 12.0) * numpy.linspace(-3.0, 3.0, 5)
if reffrequency is None:
reffrequency = [1e8]
lowcore = create_named_configuration('LOWBD2-CORE')
frequency = numpy.array(freq)
channel_bandwidth = numpy.array(chan_width)
phasecentre = SkyCoord(ra=+180.0 * u.deg,
dec=-60.0 * u.deg,
frame='icrs',
equinox='J2000')
if block:
vt = create_blockvisibility(
lowcore,
times,
frequency,
channel_bandwidth=channel_bandwidth,
weight=1.0,
phasecentre=phasecentre,
polarisation_frame=PolarisationFrame("stokesI"))
else:
vt = create_visibility(
lowcore,
times,
frequency,
channel_bandwidth=channel_bandwidth,
weight=1.0,
phasecentre=phasecentre,
polarisation_frame=PolarisationFrame("stokesI"))
cellsize = 0.001
model = create_image_from_visibility(
vt,
npixel=self.npixel,
cellsize=cellsize,
npol=1,
frequency=reffrequency,
phasecentre=phasecentre,
polarisation_frame=PolarisationFrame("stokesI"))
flux = numpy.array([[100.0]])
facets = 4
rpix = model.wcs.wcs.crpix - 1.0
spacing_pixels = self.npixel // facets
centers = [-1.5, -0.5, 0.5, 1.5]
comps = list()
for iy in centers:
for ix in centers:
p = int(round(rpix[0] + ix * spacing_pixels * numpy.sign(model.wcs.wcs.cdelt[0]))), \
int(round(rpix[1] + iy * spacing_pixels * numpy.sign(model.wcs.wcs.cdelt[1])))
sc = pixel_to_skycoord(p[0], p[1], model.wcs, origin=1)
comp = create_skycomponent(
flux=flux,
frequency=frequency,
direction=sc,
polarisation_frame=PolarisationFrame("stokesI"))
comps.append(comp)
if block:
predict_skycomponent_blockvisibility(vt, comps)
else:
predict_skycomponent_visibility(vt, comps)
insert_skycomponent(model, comps)
self.model = copy_image(model)
self.empty_model = create_empty_image_like(model)
export_image_to_fits(model, '%s/test_bags_model.fits' % (self.dir))
if add_errors:
# These will be the same for all calls
numpy.random.seed(180555)
gt = create_gaintable_from_blockvisibility(vt)
gt = simulate_gaintable(gt, phase_error=1.0, amplitude_error=0.0)
vt = apply_gaintable(vt, gt)
return vt
cellsize = advice['cellsize']
future = c.compute(
delayed(create_low_test_image_from_gleam)(
vis_graph_list[0],
npixel=npixel,
nchan=1,
cellsize=cellsize,
frequency=[frequency[0]],
channel_bandwidth=[channel_bandwidth[0]],
polarisation_frame=PolarisationFrame("stokesI")))
model = future.result()
cmodel = smooth_image(model)
export_image_to_fits(
cmodel,
'%s/imaging-low-cmodel_npixel_%d.fits' % (results_dir, npixel))
cmodel = None
predicted_vis_graph_list = create_predict_graph(vis_graph_list, model)
predicted_vis_graph_list = create_weight_vis_graph_list(
predicted_vis_graph_list, model)
predicted_vis_graph_list = compute_list(c, predicted_vis_graph_list)
# Make the dirty image and point spread function
future = c.compute(create_invert_graph(predicted_vis_graph_list,
model))
dirty, sumwt = future.result()
print("Max, min in dirty image = %.6f, %.6f, sumwt = %s" %
(dirty.data.max(), dirty.data.min(), sumwt))
export_image_to_fits(
def trial_case(results,
seed=180555,
context='wstack',
nworkers=8,
threads_per_worker=1,
processes=True,
order='frequency',
nfreqwin=7,
ntimes=3,
rmax=750.0,
facets=1,
wprojection_planes=1):
""" Single trial for performance-timings
Simulates visibilities from GLEAM including phase errors
Makes dirty image and PSF
Runs ICAL pipeline
The results are in a dictionary:
'context': input - a string describing concisely the purpose of the test
'time overall', overall execution time (s)
'time create gleam', time to create GLEAM prediction graph
'time predict', time to execute GLEAM prediction graph
'time corrupt', time to corrupt data
'time invert', time to make dirty image
'time psf invert', time to make PSF
'time ICAL graph', time to create ICAL graph
'time ICAL', time to execute ICAL graph
'context', type of imaging e.g. 'wstack'
'nworkers', number of workers to create
'threads_per_worker',
'nnodes', Number of nodes,
'processes', 'order', Ordering of data
'nfreqwin', Number of frequency windows in simulation
'ntimes', Number of hour angles in simulation
'rmax', Maximum radius of stations used in simulation (m)
'facets', Number of facets in deconvolution and imaging
'wprojection_planes', Number of wprojection planes
'vis_slices', Number of visibility slices (per Visibbility)
'npixel', Number of pixels in image
'cellsize', Cellsize in radians
'seed', Random number seed
'dirty_max', Maximum in dirty image
'dirty_min', Minimum in dirty image
'psf_max',
'psf_min',
'restored_max',
'restored_min',
'deconvolved_max',
'deconvolved_min',
'residual_max',
'residual_min',
'git_info', GIT hash (not definitive since local mods are possible)
:param results: Initial state
:param seed: Random number seed (used in gain simulations)
:param context: imaging context
:param context: Type of context: '2d'|'timeslice'|'wstack'
:param nworkers: Number of dask workers to use
:param threads_per_worker: Number of threads per worker
:param processes: Use processes instead of threads 'processes'|'threads'
:param order: See create_simulate_vis_graph
:param nfreqwin: See create_simulate_vis_graph
:param ntimes: See create_simulate_vis_graph
:param rmax: See create_simulate_vis_graph
:param facets: Number of facets to use
:param wprojection_planes: Number of wprojection planes to use
:param kwargs:
:return: results dictionary
"""
def check_workers(client, nworkers_initial):
nworkers_final = len(client.scheduler_info()['workers'])
assert nworkers_final == nworkers_initial, "Started %d workers, only %d at end" % \
(nworkers_initial, nworkers_final)
numpy.random.seed(seed)
results['seed'] = seed
start_all = time.time()
results['context'] = context
results['hostname'] = socket.gethostname()
results['git_hash'] = git_hash()
results['epoch'] = time.strftime("%Y-%m-%d %H:%M:%S")
zerow = False
print("Context is %s" % context)
results['nworkers'] = nworkers
results['threads_per_worker'] = threads_per_worker
results['processes'] = processes
results['order'] = order
results['nfreqwin'] = nfreqwin
results['ntimes'] = ntimes
results['rmax'] = rmax
results['facets'] = facets
results['wprojection_planes'] = wprojection_planes
print("At start, configuration is {0!r}".format(results))
# Parameters determining scale
frequency = numpy.linspace(0.8e8, 1.2e8, nfreqwin)
if nfreqwin > 1:
channel_bandwidth = numpy.array(nfreqwin *
[frequency[1] - frequency[0]])
else:
channel_bandwidth = numpy.array([1e6])
times = numpy.linspace(-numpy.pi / 3.0, numpy.pi / 3.0, ntimes)
phasecentre = SkyCoord(ra=+30.0 * u.deg,
dec=-60.0 * u.deg,
frame='icrs',
equinox='J2000')
vis_graph_list = create_simulate_vis_graph(
'LOWBD2',
frequency=frequency,
channel_bandwidth=channel_bandwidth,
times=times,
phasecentre=phasecentre,
order=order,
format='blockvis',
rmax=rmax)
client = get_dask_Client(n_workers=nworkers,
threads_per_worker=threads_per_worker,
processes=processes)
nworkers_initial = len(client.scheduler_info()['workers'])
check_workers(client, nworkers_initial)
results['nnodes'] = len(numpy.unique(findNodes(client)))
print("Defined %d workers on %d nodes" % (nworkers, results['nnodes']))
print("****** Visibility creation ******")
vis_graph_list = compute_list(client, vis_graph_list)
print("After creating vis_graph_list", client)
# Find the best imaging parameters.
wprojection_planes = 1
advice = advise_wide_field(convert_blockvisibility_to_visibility(
vis_graph_list[0]),
guard_band_image=6.0,
delA=0.02,
facets=facets,
wprojection_planes=wprojection_planes,
oversampling_synthesised_beam=4.0)
kernel = advice['kernel']
npixel = advice['npixels2']
cellsize = advice['cellsize']
if context == 'timeslice':
vis_slices = ntimes
elif context == '2d':
vis_slices = 1
kernel = '2d'
else:
vis_slices = advice['vis_slices']
results['vis_slices'] = vis_slices
results['cellsize'] = cellsize
results['npixel'] = npixel
gleam_model_graph = [
delayed(create_low_test_image_from_gleam)(
npixel=npixel,
frequency=[frequency[f]],
channel_bandwidth=[channel_bandwidth[f]],
cellsize=cellsize,
phasecentre=phasecentre,
polarisation_frame=PolarisationFrame("stokesI"),
flux_limit=0.1,
applybeam=True) for f, freq in enumerate(frequency)
]
start = time.time()
print("****** Starting GLEAM model creation ******")
gleam_model_graph = compute_list(client, gleam_model_graph)
cmodel = smooth_image(gleam_model_graph[0])
export_image_to_fits(cmodel, "pipelines-timings-delayed-gleam_cmodel.fits")
end = time.time()
results['time create gleam'] = end - start
print("Creating GLEAM model took %.2f seconds" % (end - start))
vis_graph_list = create_predict_graph(vis_graph_list,
gleam_model_graph,
vis_slices=51,
context=context,
kernel=kernel)
start = time.time()
print("****** Starting GLEAM model visibility prediction ******")
vis_graph_list = compute_list(client, vis_graph_list)
end = time.time()
results['time predict'] = end - start
print("After prediction", client)
print("GLEAM model Visibility prediction took %.2f seconds" %
(end - start))
# Corrupt the visibility for the GLEAM model
print("****** Visibility corruption ******")
vis_graph_list = create_corrupt_vis_graph(vis_graph_list, phase_error=1.0)
start = time.time()
vis_graph_list = compute_list(client, vis_graph_list)
end = time.time()
results['time corrupt'] = end - start
print("After corrupt", client)
print("Visibility corruption took %.2f seconds" % (end - start))
# Create an empty model image
model_graph = [
delayed(create_image_from_visibility)(
vis_graph_list[f],
npixel=npixel,
cellsize=cellsize,
frequency=[frequency[f]],
channel_bandwidth=[channel_bandwidth[f]],
polarisation_frame=PolarisationFrame("stokesI"))
for f, freq in enumerate(frequency)
]
model_graph = client.compute(model_graph, sync=True)
psf_graph = create_invert_graph(vis_graph_list,
model_graph,
vis_slices=vis_slices,
context=context,
facets=facets,
dopsf=True,
kernel=kernel)
start = time.time()
print("****** Starting PSF calculation ******")
psf, sumwt = client.compute(psf_graph, sync=True)[0]
check_workers(client, nworkers_initial)
end = time.time()
results['time psf invert'] = end - start
print("PSF invert took %.2f seconds" % (end - start))
print("After psf", client)
results['psf_max'] = qa_image(psf).data['max']
results['psf_min'] = qa_image(psf).data['min']
dirty_graph = create_invert_graph(vis_graph_list,
model_graph,
vis_slices=vis_slices,
context=context,
facets=facets,
kernel=kernel)
start = time.time()
print("****** Starting dirty image calculation ******")
dirty, sumwt = client.compute(dirty_graph, sync=True)[0]
check_workers(client, nworkers_initial)
end = time.time()
print("After dirty image", client)
results['time invert'] = end - start
print("Dirty image invert took %.2f seconds" % (end - start))
print("Maximum in dirty image is ", numpy.max(numpy.abs(dirty.data)),
", sumwt is ", sumwt)
qa = qa_image(dirty)
results['dirty_max'] = qa.data['max']
results['dirty_min'] = qa.data['min']
# Create the ICAL pipeline to run 5 major cycles, starting selfcal at cycle 1. A global solution across all
# frequencies (i.e. Visibilities) is performed.
start = time.time()
print("****** Starting ICAL ******")
start = time.time()
ical_graph = create_ical_pipeline_graph(vis_graph_list,
model_graph=model_graph,
context=context,
do_selfcal=1,
nchan=nfreqwin,
vis_slices=vis_slices,
algorithm='mmclean',
nmoments=3,
niter=1000,
fractional_threshold=0.1,
scales=[0, 3, 10],
threshold=0.1,
nmajor=5,
gain=0.7,
timeslice='auto',
global_solution=True,
window_shape='quarter')
end = time.time()
results['time ICAL graph'] = end - start
print("Construction of ICAL graph took %.2f seconds" % (end - start))
# Execute the graph
start = time.time()
result = client.compute(ical_graph, sync=True)
deconvolved, residual, restored = result
check_workers(client, nworkers_initial)
end = time.time()
print("After ICAL", client)
results['time ICAL'] = end - start
print("ICAL graph execution took %.2f seconds" % (end - start))
qa = qa_image(deconvolved[0])
results['deconvolved_max'] = qa.data['max']
results['deconvolved_min'] = qa.data['min']
export_image_to_fits(deconvolved[0],
"pipelines-timings-delayed-ical_deconvolved.fits")
qa = qa_image(residual[0][0])
results['residual_max'] = qa.data['max']
results['residual_min'] = qa.data['min']
export_image_to_fits(residual[0][0],
"pipelines-timings-delayed-ical_residual.fits")
qa = qa_image(restored[0])
results['restored_max'] = qa.data['max']
results['restored_min'] = qa.data['min']
export_image_to_fits(restored[0],
"pipelines-timings-delayed-ical_restored.fits")
#
client.shutdown()
end_all = time.time()
results['time overall'] = end_all - start_all
print("At end, results are {0!r}".format(results))
return results
