def test_deconvolve_and_restore_cube_mmclean(self): self.actualSetUp(add_errors=True) dirty_imagelist = invert_list_serial_workflow(self.vis_list, self.model_imagelist, context='2d', dopsf=False, normalize=True) psf_imagelist = invert_list_serial_workflow(self.vis_list, self.model_imagelist, context='2d', dopsf=True, normalize=True) dec_imagelist = deconvolve_list_serial_workflow( dirty_imagelist, psf_imagelist, self.model_imagelist, niter=1000, fractional_threshold=0.01, scales=[0, 3, 10], algorithm='mmclean', nmoment=3, nchan=self.freqwin, threshold=0.1, gain=0.7) residual_imagelist = residual_list_serial_workflow( self.vis_list, model_imagelist=dec_imagelist, context='2d') restored = restore_list_serial_workflow( model_imagelist=dec_imagelist, psf_imagelist=psf_imagelist, residual_imagelist=residual_imagelist, empty=self.model_imagelist)[0] export_image_to_fits( restored, '%s/test_imaging_serial_mmclean_restored.fits' % (self.dir))
def load_invert_and_deconvolve(c): v1 = create_visibility_from_ms(input_vis[0], channum=[c])[0] v2 = create_visibility_from_ms(input_vis[1], channum=[c])[0] vf = append_visibility(v1, v2) vf = convert_visibility_to_stokes(vf) vf.configuration.diameter[...] = 35.0 rows = vis_select_uvrange(vf, 0.0, uvmax=uvmax) v = create_visibility_from_rows(vf, rows) m = create_image_from_visibility(v, npixel=npixel, cellsize=cellsize, polarisation_frame=pol_frame) if context == '2d': d, sumwt = invert_2d(v, m, dopsf=False) p, sumwt = invert_2d(v, m, dopsf=True) else: d, sumwt = invert_list_serial_workflow([v], [m], context=context, dopsf=False, vis_slices=vis_slices)[0] p, sumwt = invert_list_serial_workflow([v], [m], context=context, dopsf=True, vis_slices=vis_slices)[0] c, resid = deconvolve_cube(d, p, m, threshold=0.01, fracthresh=0.01, window_shape='quarter', niter=100, gain=0.1, algorithm='hogbom-complex') r = restore_cube(c, p, resid, psfwidth=psfwidth) return r
def test_deconvolve_spectral(self): self.actualSetUp(add_errors=True) dirty_imagelist = invert_list_serial_workflow(self.vis_list, self.model_imagelist, context='2d', dopsf=False, normalize=True) psf_imagelist = invert_list_serial_workflow(self.vis_list, self.model_imagelist, context='2d', dopsf=True, normalize=True) deconvolved, _ = deconvolve_list_serial_workflow(dirty_imagelist, psf_imagelist, self.model_imagelist, niter=1000, fractional_threshold=0.1, scales=[0, 3, 10], threshold=0.1, gain=0.7) export_image_to_fits(deconvolved[0], '%s/test_imaging_serial_deconvolve_spectral.fits' % (self.dir))
def _invert_base(self, context, extra='', fluxthreshold=1.0, positionthreshold=1.0, check_components=True, facets=1, vis_slices=1, gcfcf=None, **kwargs): centre = self.freqwin // 2 dirty = invert_list_serial_workflow(self.vis_list, self.model_list, context=context, dopsf=False, normalize=True, facets=facets, vis_slices=vis_slices, gcfcf=gcfcf, **kwargs)[centre] export_image_to_fits( dirty[0], '%s/test_imaging_invert_%s%s_serial_dirty.fits' % (self.dir, context, extra)) assert numpy.max(numpy.abs(dirty[0].data)), "Image is empty" if check_components: self._checkcomponents(dirty[0], fluxthreshold, positionthreshold)
def _predict_base(self, context='2d', extra='', fluxthreshold=1.0, facets=1, vis_slices=1, gcfcf=None, **kwargs): centre = self.freqwin // 2 vis_list = zero_list_serial_workflow(self.vis_list) vis_list = predict_list_serial_workflow(vis_list, self.model_list, context=context, vis_slices=vis_slices, facets=facets, gcfcf=gcfcf, **kwargs) vis_list = subtract_list_serial_workflow(self.vis_list, vis_list) dirty = invert_list_serial_workflow(vis_list, self.model_list, context='2d', dopsf=False, gcfcf=gcfcf, normalize=True)[centre] assert numpy.max(numpy.abs(dirty[0].data)), "Residual image is empty" export_image_to_fits( dirty[0], '%s/test_imaging_predict_%s%s_serial_dirty.fits' % (self.dir, context, extra)) maxabs = numpy.max(numpy.abs(dirty[0].data)) assert maxabs < fluxthreshold, "Error %.3f greater than fluxthreshold %.3f " % ( maxabs, fluxthreshold)
def ift_ical_sm(v, sm, g): assert isinstance(v, Visibility) or isinstance(v, BlockVisibility), v assert isinstance(sm, SkyModel), sm if g is not None: assert len(g) == 2, g assert isinstance(g[0], Image), g[0] assert isinstance(g[1], ConvolutionFunction), g[1] if docal and isinstance(sm.gaintable, GainTable): if isinstance(v, Visibility): bv = convert_visibility_to_blockvisibility(v) bv = apply_gaintable(bv, sm.gaintable) v = convert_blockvisibility_to_visibility(bv) else: v = apply_gaintable(v, sm.gaintable) result = invert_list_serial_workflow([v], [sm.image], context=context, vis_slices=vis_slices, facets=facets, gcfcf=[g], **kwargs)[0] if isinstance(sm.mask, Image): result[0].data *= sm.mask.data return result
def ft_ift_sm(ov, sm, g): assert isinstance(ov, Visibility) or isinstance(ov, BlockVisibility), ov assert isinstance(sm, SkyModel), sm if g is not None: assert len(g) == 2, g assert isinstance(g[0], Image), g[0] assert isinstance(g[1], ConvolutionFunction), g[1] v = copy_visibility(ov) v.data['vis'][...] = 0.0 + 0.0j if len(sm.components) > 0: if isinstance(sm.mask, Image): comps = copy_skycomponent(sm.components) comps = apply_beam_to_skycomponent(comps, sm.mask) v = predict_skycomponent_visibility(v, comps) else: v = predict_skycomponent_visibility(v, sm.components) if isinstance(sm.image, Image): if numpy.max(numpy.abs(sm.image.data)) > 0.0: if isinstance(sm.mask, Image): model = copy_image(sm.image) model.data *= sm.mask.data else: model = sm.image v = predict_list_serial_workflow([v], [model], context=context, vis_slices=vis_slices, facets=facets, gcfcf=[g], **kwargs)[0] assert isinstance(sm.image, Image), sm.image result = invert_list_serial_workflow([v], [sm.image], context=context, vis_slices=vis_slices, facets=facets, gcfcf=[g], **kwargs)[0] if isinstance(sm.mask, Image): result[0].data *= sm.mask.data return result
def ift_ical_sm(v, sm): assert isinstance(v, Visibility), v assert isinstance(sm.image, Image), sm.image if docal and isinstance(sm.gaintable, GainTable): bv = convert_visibility_to_blockvisibility(v) bv = apply_gaintable(bv, sm.gaintable) v = convert_blockvisibility_to_visibility(bv) result = invert_list_serial_workflow([v], [sm.image], context=context, vis_slices=vis_slices, facets=facets, gcfcf=gcfcf, **kwargs)[0] if isinstance(sm.mask, Image): result[0].data *= sm.mask.data return result
def ft_ift_sm(ov, sm): assert isinstance(ov, Visibility), ov v = copy_visibility(ov) v.data['vis'][...] = 0.0 + 0.0j if len(sm.components) > 0: if isinstance(sm.mask, Image): comps = copy_skycomponent(sm.components) comps = apply_beam_to_skycomponent(comps, sm.mask) v = predict_skycomponent_visibility(v, comps) else: v = predict_skycomponent_visibility(v, sm.components) if isinstance(sm.image, Image): if numpy.max(numpy.abs(sm.image.data)) > 0.0: if isinstance(sm.mask, Image): model = copy_image(sm.image) model.data *= sm.mask.data else: model = sm.image v = predict_list_serial_workflow([v], [model], context=context, vis_slices=vis_slices, facets=facets, gcfcf=gcfcf, **kwargs)[0] assert isinstance(sm.image, Image), sm.image result = invert_list_serial_workflow([v], [sm.image], context=context, vis_slices=vis_slices, facets=facets, gcfcf=gcfcf, **kwargs)[0] if isinstance(sm.mask, Image): result[0].data *= sm.mask.data return result
def invert_list_mpi_workflow(vis_list, template_model_imagelist, context, dopsf=False, normalize=True, facets=1, vis_slices=1, gcfcf=None, comm=MPI.COMM_WORLD, **kwargs): """ Sum results from invert, iterating over the scattered image and vis_list :param vis_list: Only full for rank==0 :param template_model_imagelist: Model used to determine image parameters (in rank=0) :param dopsf: Make the PSF instead of the dirty image :param facets: Number of facets :param normalize: Normalize by sumwt :param vis_slices: Number of slices :param context: Imaging context :param gcfcg: tuple containing grid correction and convolution function (in rank=0) :param comm:MPI Communicator :param kwargs: Parameters for functions in components :return: List of (image, sumwt) tuple """ # NOTE: Be careful with normalization as normalizing parts is not the # same as normalizing the whole, normalization happens for each image in a # frequency window (in this versio we only parallelize at freqwindows def concat_tuples(list_of_tuples): if len(list_of_tuples) < 2: result_list = list_of_tuples else: result_list = list_of_tuples[0] for l in list_of_tuples[1:]: result_list += l return result_list rank = comm.Get_rank() size = comm.Get_size() log.info( '%d: In invert_list_mpi_workflow: %d elements in vis_list %d in model' % (rank, len(vis_list), len(template_model_imagelist))) assert get_parameter(kwargs, "use_serial_invert", True), "Only freq paralellization implemented" #if get_parameter(kwargs, "use_serial_invert", False): if get_parameter(kwargs, "use_serial_invert", True): from workflows.serial.imaging.imaging_serial import invert_list_serial_workflow results_vislist = list() # Distribute visibilities and model by freq sub_vis_list = numpy.array_split(vis_list, size) sub_vis_list = comm.scatter(sub_vis_list, root=0) sub_template_model_imagelist = numpy.array_split( template_model_imagelist, size) sub_template_model_imagelist = comm.scatter( sub_template_model_imagelist, root=0) if gcfcf is not None: sub_gcfcf = numpy.array_split(gcfcf, size) sub_gcfcf = comm.scatter(sub_gcfcf, root=0) isinstance(sub_vis_list[0], Visibility) sub_results_vislist = [ invert_list_serial_workflow( vis_list=[sub_vis_list[i]], template_model_imagelist=[sub_template_model_imagelist[i]], context=context, dopsf=dopsf, normalize=normalize, vis_slices=vis_slices, facets=facets, gcfcf=gcfcf, **kwargs)[0] for i, _ in enumerate(sub_vis_list) ] #print("%d sub_results_vislist" %rank,sub_results_vislist) results_vislist = comm.gather(sub_results_vislist, root=0) #print("%d results_vislist before concatenate"%rank,results_vislist) if rank == 0: #image_results_list_list=[x for x in image_results_list_list if x] #results_vislist=numpy.concatenate(results_vislist) # TODO: concatenate dos not concatenate well a list of tuples # it returns a 2d array instead of a concatenated list of tuples results_vislist = concat_tuples(results_vislist) else: results_vislist = list() #print("%d results_vislist"%rank,results_vislist) return results_vislist
#model_list=numpy.concatenate(model_list) model_list = concat_tuples(model_list) # In[ ]: else: model_list = list() log.debug('%d model_list len %d' % (rank, len(model_list))) log.info('%d: About to start invert' % (rank)) print('%d: About to start invert' % (rank), flush=True) start = time.time() original_invert = False if original_invert: if rank == 0: dirty_list = invert_list_serial_workflow(predicted_vislist, model_list, context='wstack', vis_slices=vis_slices, dopsf=False) psf_list = invert_list_serial_workflow(predicted_vislist, model_list, context='wstack', vis_slices=vis_slices, dopsf=True) else: dirty_list = invert_list_mpi_workflow(predicted_vislist, model_list, context='wstack', vis_slices=vis_slices, dopsf=False) psf_list = invert_list_mpi_workflow(predicted_vislist, model_list,
for vt in vis_list: channel_model = create_image_from_visibility( vt, cellsize=cellsize, npixel=512, nchan=1, imagecentre=vis_list[0].phasecentre, polarisation_frame=PolarisationFrame('stokesIQUV')) beam = create_pb(channel_model, telescope='VLA', pointingcentre=vt.phasecentre, use_local=False) beam.data /= numpy.max(beam.data) dirty, sumwt = invert_list_serial_workflow([vt], [channel_model])[0] print(sumwt) mosaic.data += dirty.data * beam.data mosaicsens.data += beam.data**2 show_image(dirty) plt.show() show_image(mosaic, cm='Greys', title='Linear mosaic') plt.show() show_image(mosaicsens, cm='Greys', title='Linear mosaic sensitivity') plt.show() from wrappers.serial.image.operations import export_image_to_fits export_image_to_fits(mosaic, "mosaics.fits") export_image_to_fits(mosaicsens, "mosaicsens.fits")
gt = simulate_gaintable(gt, phase_error=1.0) #print("np.sum(gt.data): ", numpy.sum(gt.data['gain'])) blockvis = apply_gaintable(block_vis, gt) #print("np.sum(blockvis.data): ", numpy.sum(blockvis.data['vis'])) model = create_image_from_visibility(block_vis, npixel=npixel, frequency=[numpy.average(frequency)], nchan=1, channel_bandwidth=[numpy.sum(channel_bandwidth)], cellsize=cellsize, phasecentre=phasecentre) #print("model sum, min, max, shape: ", numpy.sum(model.data), numpy.amin(model.data), numpy.amax(model.data), model.shape) print(qa_image(model, context='Blockvis model image')) export_image_to_fits(model, '%s/imaging-blockvis_model.fits' % (results_dir)) dirty, sumwt = invert_list_serial_workflow(predicted_vis, model, vis_slices=vis_slices, dopsf=False, context='wstack') print(qa_image(dirty, context='Dirty image')) export_image_to_fits(dirty, '%s/imaging-dirty.fits' % (results_dir)) deconvolved, residual, restored = ical_list_serial_workflow(vis_list=[blockvis], model_imagelist=[model], vis_slices=vis_slices, timeslice='auto', algorithm='hogbom', niter=1000, fractional_threshold=0.1, threshold=0.1, context='wstack', nmajor=5, gain=0.1, first_selfcal=1, global_solution=False) print(qa_image(deconvolved, context='Clean image')) export_image_to_fits(deconvolved, '%s/imaging-dask_ical_deconvolved.fits'
results_dir = arl_path('test_results') # Test requires that casa be installed try: bvt = create_blockvisibility_from_ms(arl_path('data/vis/sim-2.ms'), channum=[35, 36, 37, 38, 39])[0] bvt.configuration.diameter[...] = 35.0 vt = convert_blockvisibility_to_visibility(bvt) vt = convert_visibility_to_stokes(vt) cellsize = 20.0 * numpy.pi / (180.0 * 3600.0) npixel = 512 model = create_image_from_visibility(vt, cellsize=cellsize, npixel=npixel, polarisation_frame=PolarisationFrame('stokesIQUV')) dirty, sumwt = invert_list_serial_workflow([vt], [model], context='2d')[0] psf, sumwt = invert_list_serial_workflow([vt], [model], context='2d', dopsf=True)[0] export_image_to_fits(dirty, '%s/compare_imaging_sim2_dirty.fits' % (results_dir)) export_image_to_fits(psf, '%s/compare_imaging_sim2_psf.fits' % (results_dir)) # Deconvolve using clean comp, residual = deconvolve_cube(dirty, psf, niter=10000, threshold=0.001, fractional_threshold=0.001, window_shape='quarter', gain=0.7, scales=[0, 3, 10, 30]) restored = restore_cube(comp, psf, residual) export_image_to_fits(restored, '%s/compare_imaging_sim2_restored.fits' % (results_dir)) export_image_to_fits(residual, '%s/compare_imaging_sim2_residual.fits' % (results_dir)) qa = qa_image(restored) assert numpy.abs(qa.data['max'] - 1.006140596404203) < 1e-7, qa