def test_deconvolve_and_restore_cube_mmclean(self):
self.actualSetUp(add_errors=True)
dirty_imagelist = invert_list_serial_workflow(self.vis_list,
self.model_imagelist,
context='2d',
dopsf=False,
normalize=True)
psf_imagelist = invert_list_serial_workflow(self.vis_list,
self.model_imagelist,
context='2d',
dopsf=True,
normalize=True)
dec_imagelist = deconvolve_list_serial_workflow(
dirty_imagelist,
psf_imagelist,
self.model_imagelist,
niter=1000,
fractional_threshold=0.01,
scales=[0, 3, 10],
algorithm='mmclean',
nmoment=3,
nchan=self.freqwin,
threshold=0.1,
gain=0.7)
residual_imagelist = residual_list_serial_workflow(
self.vis_list, model_imagelist=dec_imagelist, context='2d')
restored = restore_list_serial_workflow(
model_imagelist=dec_imagelist,
psf_imagelist=psf_imagelist,
residual_imagelist=residual_imagelist,
empty=self.model_imagelist)[0]
export_image_to_fits(
restored,
'%s/test_imaging_serial_mmclean_restored.fits' % (self.dir))
def load_invert_and_deconvolve(c):
v1 = create_visibility_from_ms(input_vis[0], channum=[c])[0]
v2 = create_visibility_from_ms(input_vis[1], channum=[c])[0]
vf = append_visibility(v1, v2)
vf = convert_visibility_to_stokes(vf)
vf.configuration.diameter[...] = 35.0
rows = vis_select_uvrange(vf, 0.0, uvmax=uvmax)
v = create_visibility_from_rows(vf, rows)
m = create_image_from_visibility(v, npixel=npixel, cellsize=cellsize,
polarisation_frame=pol_frame)
if context == '2d':
d, sumwt = invert_2d(v, m, dopsf=False)
p, sumwt = invert_2d(v, m, dopsf=True)
else:
d, sumwt = invert_list_serial_workflow([v], [m], context=context, dopsf=False,
vis_slices=vis_slices)[0]
p, sumwt = invert_list_serial_workflow([v], [m], context=context, dopsf=True,
vis_slices=vis_slices)[0]
c, resid = deconvolve_cube(d, p, m, threshold=0.01, fracthresh=0.01, window_shape='quarter',
niter=100, gain=0.1, algorithm='hogbom-complex')
r = restore_cube(c, p, resid, psfwidth=psfwidth)
return r
def test_deconvolve_spectral(self):
self.actualSetUp(add_errors=True)
dirty_imagelist = invert_list_serial_workflow(self.vis_list, self.model_imagelist,
context='2d',
dopsf=False, normalize=True)
psf_imagelist = invert_list_serial_workflow(self.vis_list, self.model_imagelist,
context='2d',
dopsf=True, normalize=True)
deconvolved, _ = deconvolve_list_serial_workflow(dirty_imagelist, psf_imagelist, self.model_imagelist, niter=1000,
fractional_threshold=0.1, scales=[0, 3, 10],
threshold=0.1, gain=0.7)
export_image_to_fits(deconvolved[0], '%s/test_imaging_serial_deconvolve_spectral.fits' %
(self.dir))
def _invert_base(self,
context,
extra='',
fluxthreshold=1.0,
positionthreshold=1.0,
check_components=True,
facets=1,
vis_slices=1,
gcfcf=None,
**kwargs):
centre = self.freqwin // 2
dirty = invert_list_serial_workflow(self.vis_list,
self.model_list,
context=context,
dopsf=False,
normalize=True,
facets=facets,
vis_slices=vis_slices,
gcfcf=gcfcf,
**kwargs)[centre]
export_image_to_fits(
dirty[0], '%s/test_imaging_invert_%s%s_serial_dirty.fits' %
(self.dir, context, extra))
assert numpy.max(numpy.abs(dirty[0].data)), "Image is empty"
if check_components:
self._checkcomponents(dirty[0], fluxthreshold, positionthreshold)
def _predict_base(self,
context='2d',
extra='',
fluxthreshold=1.0,
facets=1,
vis_slices=1,
gcfcf=None,
**kwargs):
centre = self.freqwin // 2
vis_list = zero_list_serial_workflow(self.vis_list)
vis_list = predict_list_serial_workflow(vis_list,
self.model_list,
context=context,
vis_slices=vis_slices,
facets=facets,
gcfcf=gcfcf,
**kwargs)
vis_list = subtract_list_serial_workflow(self.vis_list, vis_list)
dirty = invert_list_serial_workflow(vis_list,
self.model_list,
context='2d',
dopsf=False,
gcfcf=gcfcf,
normalize=True)[centre]
assert numpy.max(numpy.abs(dirty[0].data)), "Residual image is empty"
export_image_to_fits(
dirty[0], '%s/test_imaging_predict_%s%s_serial_dirty.fits' %
(self.dir, context, extra))
maxabs = numpy.max(numpy.abs(dirty[0].data))
assert maxabs < fluxthreshold, "Error %.3f greater than fluxthreshold %.3f " % (
maxabs, fluxthreshold)
def ift_ical_sm(v, sm, g):
assert isinstance(v, Visibility) or isinstance(v, BlockVisibility), v
assert isinstance(sm, SkyModel), sm
if g is not None:
assert len(g) == 2, g
assert isinstance(g[0], Image), g[0]
assert isinstance(g[1], ConvolutionFunction), g[1]
if docal and isinstance(sm.gaintable, GainTable):
if isinstance(v, Visibility):
bv = convert_visibility_to_blockvisibility(v)
bv = apply_gaintable(bv, sm.gaintable)
v = convert_blockvisibility_to_visibility(bv)
else:
v = apply_gaintable(v, sm.gaintable)
result = invert_list_serial_workflow([v], [sm.image],
context=context,
vis_slices=vis_slices,
facets=facets,
gcfcf=[g],
**kwargs)[0]
if isinstance(sm.mask, Image):
result[0].data *= sm.mask.data
return result
def ft_ift_sm(ov, sm, g):
assert isinstance(ov, Visibility) or isinstance(ov,
BlockVisibility), ov
assert isinstance(sm, SkyModel), sm
if g is not None:
assert len(g) == 2, g
assert isinstance(g[0], Image), g[0]
assert isinstance(g[1], ConvolutionFunction), g[1]
v = copy_visibility(ov)
v.data['vis'][...] = 0.0 + 0.0j
if len(sm.components) > 0:
if isinstance(sm.mask, Image):
comps = copy_skycomponent(sm.components)
comps = apply_beam_to_skycomponent(comps, sm.mask)
v = predict_skycomponent_visibility(v, comps)
else:
v = predict_skycomponent_visibility(v, sm.components)
if isinstance(sm.image, Image):
if numpy.max(numpy.abs(sm.image.data)) > 0.0:
if isinstance(sm.mask, Image):
model = copy_image(sm.image)
model.data *= sm.mask.data
else:
model = sm.image
v = predict_list_serial_workflow([v], [model],
context=context,
vis_slices=vis_slices,
facets=facets,
gcfcf=[g],
**kwargs)[0]
assert isinstance(sm.image, Image), sm.image
result = invert_list_serial_workflow([v], [sm.image],
context=context,
vis_slices=vis_slices,
facets=facets,
gcfcf=[g],
**kwargs)[0]
if isinstance(sm.mask, Image):
result[0].data *= sm.mask.data
return result
def ift_ical_sm(v, sm):
assert isinstance(v, Visibility), v
assert isinstance(sm.image, Image), sm.image
if docal and isinstance(sm.gaintable, GainTable):
bv = convert_visibility_to_blockvisibility(v)
bv = apply_gaintable(bv, sm.gaintable)
v = convert_blockvisibility_to_visibility(bv)
result = invert_list_serial_workflow([v], [sm.image],
context=context,
vis_slices=vis_slices,
facets=facets,
gcfcf=gcfcf,
**kwargs)[0]
if isinstance(sm.mask, Image):
result[0].data *= sm.mask.data
return result
def ft_ift_sm(ov, sm):
assert isinstance(ov, Visibility), ov
v = copy_visibility(ov)
v.data['vis'][...] = 0.0 + 0.0j
if len(sm.components) > 0:
if isinstance(sm.mask, Image):
comps = copy_skycomponent(sm.components)
comps = apply_beam_to_skycomponent(comps, sm.mask)
v = predict_skycomponent_visibility(v, comps)
else:
v = predict_skycomponent_visibility(v, sm.components)
if isinstance(sm.image, Image):
if numpy.max(numpy.abs(sm.image.data)) > 0.0:
if isinstance(sm.mask, Image):
model = copy_image(sm.image)
model.data *= sm.mask.data
else:
model = sm.image
v = predict_list_serial_workflow([v], [model],
context=context,
vis_slices=vis_slices,
facets=facets,
gcfcf=gcfcf,
**kwargs)[0]
assert isinstance(sm.image, Image), sm.image
result = invert_list_serial_workflow([v], [sm.image],
context=context,
vis_slices=vis_slices,
facets=facets,
gcfcf=gcfcf,
**kwargs)[0]
if isinstance(sm.mask, Image):
result[0].data *= sm.mask.data
return result
def invert_list_mpi_workflow(vis_list,
template_model_imagelist,
context,
dopsf=False,
normalize=True,
facets=1,
vis_slices=1,
gcfcf=None,
comm=MPI.COMM_WORLD,
**kwargs):
""" Sum results from invert, iterating over the scattered image and vis_list
:param vis_list: Only full for rank==0
:param template_model_imagelist: Model used to determine image parameters
(in rank=0)
:param dopsf: Make the PSF instead of the dirty image
:param facets: Number of facets
:param normalize: Normalize by sumwt
:param vis_slices: Number of slices
:param context: Imaging context
:param gcfcg: tuple containing grid correction and convolution function (in
rank=0)
:param comm:MPI Communicator
:param kwargs: Parameters for functions in components
:return: List of (image, sumwt) tuple
"""
# NOTE: Be careful with normalization as normalizing parts is not the
# same as normalizing the whole, normalization happens for each image in a
# frequency window (in this versio we only parallelize at freqwindows
def concat_tuples(list_of_tuples):
if len(list_of_tuples) < 2:
result_list = list_of_tuples
else:
result_list = list_of_tuples[0]
for l in list_of_tuples[1:]:
result_list += l
return result_list
rank = comm.Get_rank()
size = comm.Get_size()
log.info(
'%d: In invert_list_mpi_workflow: %d elements in vis_list %d in model'
% (rank, len(vis_list), len(template_model_imagelist)))
assert get_parameter(kwargs, "use_serial_invert",
True), "Only freq paralellization implemented"
#if get_parameter(kwargs, "use_serial_invert", False):
if get_parameter(kwargs, "use_serial_invert", True):
from workflows.serial.imaging.imaging_serial import invert_list_serial_workflow
results_vislist = list()
# Distribute visibilities and model by freq
sub_vis_list = numpy.array_split(vis_list, size)
sub_vis_list = comm.scatter(sub_vis_list, root=0)
sub_template_model_imagelist = numpy.array_split(
template_model_imagelist, size)
sub_template_model_imagelist = comm.scatter(
sub_template_model_imagelist, root=0)
if gcfcf is not None:
sub_gcfcf = numpy.array_split(gcfcf, size)
sub_gcfcf = comm.scatter(sub_gcfcf, root=0)
isinstance(sub_vis_list[0], Visibility)
sub_results_vislist = [
invert_list_serial_workflow(
vis_list=[sub_vis_list[i]],
template_model_imagelist=[sub_template_model_imagelist[i]],
context=context,
dopsf=dopsf,
normalize=normalize,
vis_slices=vis_slices,
facets=facets,
gcfcf=gcfcf,
**kwargs)[0] for i, _ in enumerate(sub_vis_list)
]
#print("%d sub_results_vislist" %rank,sub_results_vislist)
results_vislist = comm.gather(sub_results_vislist, root=0)
#print("%d results_vislist before concatenate"%rank,results_vislist)
if rank == 0:
#image_results_list_list=[x for x in image_results_list_list if x]
#results_vislist=numpy.concatenate(results_vislist)
# TODO: concatenate dos not concatenate well a list of tuples
# it returns a 2d array instead of a concatenated list of tuples
results_vislist = concat_tuples(results_vislist)
else:
results_vislist = list()
#print("%d results_vislist"%rank,results_vislist)
return results_vislist
#model_list=numpy.concatenate(model_list)
model_list = concat_tuples(model_list)
# In[ ]:
else:
model_list = list()
log.debug('%d model_list len %d' % (rank, len(model_list)))
log.info('%d: About to start invert' % (rank))
print('%d: About to start invert' % (rank), flush=True)
start = time.time()
original_invert = False
if original_invert:
if rank == 0:
dirty_list = invert_list_serial_workflow(predicted_vislist,
model_list,
context='wstack',
vis_slices=vis_slices,
dopsf=False)
psf_list = invert_list_serial_workflow(predicted_vislist,
model_list,
context='wstack',
vis_slices=vis_slices,
dopsf=True)
else:
dirty_list = invert_list_mpi_workflow(predicted_vislist,
model_list,
context='wstack',
vis_slices=vis_slices,
dopsf=False)
psf_list = invert_list_mpi_workflow(predicted_vislist,
model_list,
for vt in vis_list:
channel_model = create_image_from_visibility(
vt,
cellsize=cellsize,
npixel=512,
nchan=1,
imagecentre=vis_list[0].phasecentre,
polarisation_frame=PolarisationFrame('stokesIQUV'))
beam = create_pb(channel_model,
telescope='VLA',
pointingcentre=vt.phasecentre,
use_local=False)
beam.data /= numpy.max(beam.data)
dirty, sumwt = invert_list_serial_workflow([vt], [channel_model])[0]
print(sumwt)
mosaic.data += dirty.data * beam.data
mosaicsens.data += beam.data**2
show_image(dirty)
plt.show()
show_image(mosaic, cm='Greys', title='Linear mosaic')
plt.show()
show_image(mosaicsens, cm='Greys', title='Linear mosaic sensitivity')
plt.show()
from wrappers.serial.image.operations import export_image_to_fits
export_image_to_fits(mosaic, "mosaics.fits")
export_image_to_fits(mosaicsens, "mosaicsens.fits")
gt = simulate_gaintable(gt, phase_error=1.0)
#print("np.sum(gt.data): ", numpy.sum(gt.data['gain']))
blockvis = apply_gaintable(block_vis, gt)
#print("np.sum(blockvis.data): ", numpy.sum(blockvis.data['vis']))
model = create_image_from_visibility(block_vis, npixel=npixel, frequency=[numpy.average(frequency)], nchan=1,
channel_bandwidth=[numpy.sum(channel_bandwidth)], cellsize=cellsize, phasecentre=phasecentre)
#print("model sum, min, max, shape: ", numpy.sum(model.data), numpy.amin(model.data), numpy.amax(model.data), model.shape)
print(qa_image(model, context='Blockvis model image'))
export_image_to_fits(model, '%s/imaging-blockvis_model.fits'
% (results_dir))
dirty, sumwt = invert_list_serial_workflow(predicted_vis, model, vis_slices=vis_slices, dopsf=False, context='wstack')
print(qa_image(dirty, context='Dirty image'))
export_image_to_fits(dirty, '%s/imaging-dirty.fits'
% (results_dir))
deconvolved, residual, restored = ical_list_serial_workflow(vis_list=[blockvis],
model_imagelist=[model], vis_slices=vis_slices,
timeslice='auto',
algorithm='hogbom', niter=1000, fractional_threshold=0.1, threshold=0.1,
context='wstack', nmajor=5, gain=0.1, first_selfcal=1,
global_solution=False)
print(qa_image(deconvolved, context='Clean image'))
export_image_to_fits(deconvolved, '%s/imaging-dask_ical_deconvolved.fits'
results_dir = arl_path('test_results')
# Test requires that casa be installed
try:
bvt = create_blockvisibility_from_ms(arl_path('data/vis/sim-2.ms'), channum=[35, 36, 37, 38, 39])[0]
bvt.configuration.diameter[...] = 35.0
vt = convert_blockvisibility_to_visibility(bvt)
vt = convert_visibility_to_stokes(vt)
cellsize = 20.0 * numpy.pi / (180.0 * 3600.0)
npixel = 512
model = create_image_from_visibility(vt, cellsize=cellsize, npixel=npixel,
polarisation_frame=PolarisationFrame('stokesIQUV'))
dirty, sumwt = invert_list_serial_workflow([vt], [model], context='2d')[0]
psf, sumwt = invert_list_serial_workflow([vt], [model], context='2d', dopsf=True)[0]
export_image_to_fits(dirty, '%s/compare_imaging_sim2_dirty.fits' % (results_dir))
export_image_to_fits(psf, '%s/compare_imaging_sim2_psf.fits' % (results_dir))
# Deconvolve using clean
comp, residual = deconvolve_cube(dirty, psf, niter=10000, threshold=0.001, fractional_threshold=0.001,
window_shape='quarter', gain=0.7, scales=[0, 3, 10, 30])
restored = restore_cube(comp, psf, residual)
export_image_to_fits(restored, '%s/compare_imaging_sim2_restored.fits' % (results_dir))
export_image_to_fits(residual, '%s/compare_imaging_sim2_residual.fits' % (results_dir))
qa = qa_image(restored)
assert numpy.abs(qa.data['max'] - 1.006140596404203) < 1e-7, qa
