def _predict_base(self, context='2d', extra='', fluxthreshold=1.0, facets=1, vis_slices=1, gcfcf=None, **kwargs): centre = self.freqwin // 2 vis_list = zero_list_serial_workflow(self.vis_list) vis_list = predict_list_serial_workflow(vis_list, self.model_list, context=context, vis_slices=vis_slices, facets=facets, gcfcf=gcfcf, **kwargs) vis_list = subtract_list_serial_workflow(self.vis_list, vis_list) dirty = invert_list_serial_workflow(vis_list, self.model_list, context='2d', dopsf=False, gcfcf=gcfcf, normalize=True)[centre] assert numpy.max(numpy.abs(dirty[0].data)), "Residual image is empty" export_image_to_fits( dirty[0], '%s/test_imaging_predict_%s%s_serial_dirty.fits' % (self.dir, context, extra)) maxabs = numpy.max(numpy.abs(dirty[0].data)) assert maxabs < fluxthreshold, "Error %.3f greater than fluxthreshold %.3f " % ( maxabs, fluxthreshold)
def ft_cal_sm(ov, sm): assert isinstance(ov, Visibility), ov assert isinstance(sm, SkyModel), sm v = copy_visibility(ov) v.data['vis'][...] = 0.0 + 0.0j if len(sm.components) > 0: if isinstance(sm.mask, Image): comps = copy_skycomponent(sm.components) comps = apply_beam_to_skycomponent(comps, sm.mask) v = predict_skycomponent_visibility(v, comps) else: v = predict_skycomponent_visibility(v, sm.components) if isinstance(sm.image, Image): if numpy.max(numpy.abs(sm.image.data)) > 0.0: if isinstance(sm.mask, Image): model = copy_image(sm.image) model.data *= sm.mask.data else: model = sm.image v = predict_list_serial_workflow([v], [model], context=context, vis_slices=vis_slices, facets=facets, gcfcf=gcfcf, **kwargs)[0] if docal and isinstance(sm.gaintable, GainTable): bv = convert_visibility_to_blockvisibility(v) bv = apply_gaintable(bv, sm.gaintable, inverse=True) v = convert_blockvisibility_to_visibility(bv) return v
def predict_list_mpi_workflow(vis_list, model_imagelist, context, vis_slices=1, facets=1, gcfcf=None, comm=MPI.COMM_WORLD, **kwargs): """Predict, iterating over both the scattered vis_list and image The visibility and image are scattered, the visibility is predicted on each part, and then the parts are assembled. About data distribution: vis_list and model_imagelist live in rank 0; vis_slices, facets, context are replicated in all nodes. gcfcf if exists lives in rank 0; if not every mpi proc will create its own for the corresponding subset of the image. :param vis_list: :param model_imagelist: Model used to determine image parameters :param vis_slices: Number of vis slices (w stack or timeslice) :param facets: Number of facets (per axis) :param context: Type of processing e.g. 2d, wstack, timeslice or facets :param gcfcg: tuple containing grid correction and convolution function :param comm: MPI communicator :param kwargs: Parameters for functions in components :return: List of vis_lists """ rank = comm.Get_rank() size = comm.Get_size() log.info('%d: In predict_list_mpi_workflow: %d elements in vis_list' % (rank,len(vis_list))) # the assert only makes sense in proc 0 as for the others both lists are # empty assert len(vis_list) == len(model_imagelist), "Model must be the same length as the vis_list" # The use_serial_predict version paralelizes by freq (my opt version) assert get_parameter(kwargs, "use_serial_predict", True),"Only freq paralellization implemented" #if get_parameter(kwargs, "use_serial_predict", False): if get_parameter(kwargs, "use_serial_predict", True): from workflows.serial.imaging.imaging_serial import predict_list_serial_workflow image_results_list = list() # Distribute visibilities and model by freq sub_vis_list= numpy.array_split(vis_list, size) sub_vis_list=comm.scatter(sub_vis_list,root=0) sub_model_imagelist= numpy.array_split(model_imagelist, size) sub_model_imagelist=comm.scatter(sub_model_imagelist,root=0) if gcfcf is not None: sub_gcfcf = numpy.array_split(gcfcf,size) sub_gcfcf=comm.scatter(sub_gcfcf,root=0) isinstance(sub_vis_list[0], Visibility) image_results_list= [predict_list_serial_workflow(vis_list=[sub_vis_list[i]], model_imagelist=[sub_model_imagelist[i]], vis_slices=vis_slices, facets=facets, context=context, gcfcf=gcfcf, **kwargs)[0] for i, _ in enumerate(sub_vis_list)] #print(image_results_list) image_results_list=comm.gather(image_results_list,root=0) if rank == 0: #image_results_list_list=[x for x in image_results_list_list if x] image_results_list=numpy.concatenate(image_results_list) else: image_results_list=list() return image_results_list
def predict_skymodel_list_serial_workflow(vis_list, skymodel_list, context, vis_slices=1, facets=1, gcfcf=None, **kwargs): """Predict from a skymodel, iterating over both the vis_list and skymodel The visibility and image are scattered, the visibility is predicted on each part, and then the parts are assembled. :param vis_list: :param skymodel_list: skymodel list :param vis_slices: Number of vis slices (w stack or timeslice) :param facets: Number of facets (per axis) :param context: Type of processing e.g. 2d, wstack, timeslice or facets :param gcfcg: tuple containing grid correction and convolution function :param kwargs: Parameters for functions in components :return: List of vis_lists """ def extract_comps(sm): return sm.components def extract_image(sm): return sm.images[0] comp = [extract_comps(sm) for sm in skymodel_list] images = [extract_image(sm) for sm in skymodel_list] dft_vis_list = zero_list_serial_workflow(vis_list) dft_vis_list = [ predict_skycomponent_visibility(dft_vis_list[i], comp[i]) for i, _ in enumerate(dft_vis_list) ] fft_vis_list = zero_list_serial_workflow(vis_list) fft_vis_list = predict_list_serial_workflow(fft_vis_list, images, context=context, vis_slices=vis_slices, facets=facets, gcfcf=gcfcf, **kwargs) def vis_add(v1, v2): vout = copy_visibility(v1) vout.data['vis'] += v2.data['vis'] return vout return [ vis_add(dft_vis_list[i], fft_vis_list[i]) for i, _ in enumerate(dft_vis_list) ]
def ft_ift_sm(ov, sm, g): assert isinstance(ov, Visibility) or isinstance(ov, BlockVisibility), ov assert isinstance(sm, SkyModel), sm if g is not None: assert len(g) == 2, g assert isinstance(g[0], Image), g[0] assert isinstance(g[1], ConvolutionFunction), g[1] v = copy_visibility(ov) v.data['vis'][...] = 0.0 + 0.0j if len(sm.components) > 0: if isinstance(sm.mask, Image): comps = copy_skycomponent(sm.components) comps = apply_beam_to_skycomponent(comps, sm.mask) v = predict_skycomponent_visibility(v, comps) else: v = predict_skycomponent_visibility(v, sm.components) if isinstance(sm.image, Image): if numpy.max(numpy.abs(sm.image.data)) > 0.0: if isinstance(sm.mask, Image): model = copy_image(sm.image) model.data *= sm.mask.data else: model = sm.image v = predict_list_serial_workflow([v], [model], context=context, vis_slices=vis_slices, facets=facets, gcfcf=[g], **kwargs)[0] assert isinstance(sm.image, Image), sm.image result = invert_list_serial_workflow([v], [sm.image], context=context, vis_slices=vis_slices, facets=facets, gcfcf=[g], **kwargs)[0] if isinstance(sm.mask, Image): result[0].data *= sm.mask.data return result
def ft_ift_sm(ov, sm): assert isinstance(ov, Visibility), ov v = copy_visibility(ov) v.data['vis'][...] = 0.0 + 0.0j if len(sm.components) > 0: if isinstance(sm.mask, Image): comps = copy_skycomponent(sm.components) comps = apply_beam_to_skycomponent(comps, sm.mask) v = predict_skycomponent_visibility(v, comps) else: v = predict_skycomponent_visibility(v, sm.components) if isinstance(sm.image, Image): if numpy.max(numpy.abs(sm.image.data)) > 0.0: if isinstance(sm.mask, Image): model = copy_image(sm.image) model.data *= sm.mask.data else: model = sm.image v = predict_list_serial_workflow([v], [model], context=context, vis_slices=vis_slices, facets=facets, gcfcf=gcfcf, **kwargs)[0] assert isinstance(sm.image, Image), sm.image result = invert_list_serial_workflow([v], [sm.image], context=context, vis_slices=vis_slices, facets=facets, gcfcf=gcfcf, **kwargs)[0] if isinstance(sm.mask, Image): result[0].data *= sm.mask.data return result
# each freqw during predict, it would safe time but use more space gleam_model = comm.gather(sub_gleam_model, root=0) if rank == 0: gleam_model = numpy.concatenate(gleam_model) else: gleam_model = list() # In[ ]: original_predict = False if original_predict: if rank == 0: log.info('About to run predict to get predicted visibility') predicted_vislist = predict_list_serial_workflow(vis_list, gleam_model, context='wstack', vis_slices=vis_slices) else: log.info('%d: About to run predict to get predicted visibility' % (rank)) print('%d: About to run predict to get predicted visibility' % (rank), flush=True) start = time.time() # All procs call the function but only rank=0 gets the predicted_vislist predicted_vislist = predict_list_mpi_workflow(vis_list, gleam_model, context='wstack', vis_slices=vis_slices) end = time.time() #log.info('About to run corrupt to get corrupted visibility') #corrupted_vislist = corrupt_list_serial_workflow(predicted_vislist, phase_error=1.0)
wprojection_planes=1 advice=advise_wide_field(block_vis, guard_band_image=4.0, delA=0.02, wprojection_planes=wprojection_planes) vis_slices = advice['vis_slices'] npixel=advice['npixels2'] cellsize=advice['cellsize'] gleam_model = create_low_test_image_from_gleam(npixel=npixel, frequency=frequency, channel_bandwidth=channel_bandwidth, cellsize=cellsize, phasecentre=phasecentre, flux_limit = 1.0, applybeam=True) predicted_vis = predict_list_serial_workflow(block_vis, gleam_model, vis_slices=51, context='wstack') #print("np.sum(predicted_vis.data): ", numpy.sum(predicted_vis.data['vis'])) block_vis=convert_visibility_to_blockvisibility(predicted_vis) #print("np.sum(block_vis.data): ", numpy.sum(block_vis.data['vis'])) #print("nchan npol nants ", block_vis.nchan, block_vis.npol, block_vis.nants) #print("uvw", block_vis.uvw, numpy.sum(block_vis.uvw)) #print("vis", block_vis.vis, numpy.sum(block_vis.vis)) #print("weight", block_vis.weight, numpy.sum(block_vis.weight)) #print("time", block_vis.time, numpy.sum(block_vis.time)) #print("integration_time", block_vis.integration_time, numpy.sum(block_vis.integration_time)) #print("nvis, size", block_vis.nvis, block_vis.size()) gt = create_gaintable_from_blockvisibility(block_vis) #print("np.sum(gt.data): ", numpy.sum(gt.data['gain'])) gt = simulate_gaintable(gt, phase_error=1.0)
def test_export_ms(self): if run_ms_tests == False: return msoutfile = arl_path("data/vis/Test_output.ms") from astropy.coordinates import SkyCoord from astropy import units as u from wrappers.serial.image.operations import show_image, export_image_to_fits from wrappers.serial.simulation.configurations import create_named_configuration from wrappers.serial.simulation.testing_support import create_test_image from wrappers.serial.imaging.base import create_image_from_visibility from wrappers.serial.imaging.base import advise_wide_field from workflows.serial.imaging.imaging_serial import invert_list_serial_workflow, predict_list_serial_workflow from data_models.polarisation import PolarisationFrame lowr3 = create_named_configuration('LOWBD2', rmax=750.0) times = numpy.zeros([1]) frequency = numpy.array([1e8]) channelbandwidth = numpy.array([1e6]) phasecentre = SkyCoord(ra=+15.0 * u.deg, dec=-45.0 * u.deg, frame='icrs', equinox='J2000') bvis = create_blockvisibility(lowr3, times, frequency, phasecentre=phasecentre, weight=1.0, polarisation_frame=PolarisationFrame('stokesI'), channel_bandwidth=channelbandwidth) vt = convert_blockvisibility_to_visibility(bvis) advice = advise_wide_field(vt, guard_band_image=3.0, delA=0.1, facets=1, wprojection_planes=1, oversampling_synthesised_beam=4.0) cellsize = advice['cellsize'] m31image = create_test_image(frequency=frequency, cellsize=cellsize) nchan, npol, ny, nx = m31image.data.shape m31image.wcs.wcs.crval[0] = vt.phasecentre.ra.deg m31image.wcs.wcs.crval[1] = vt.phasecentre.dec.deg m31image.wcs.wcs.crpix[0] = float(nx // 2) m31image.wcs.wcs.crpix[1] = float(ny // 2) vt = predict_list_serial_workflow([vt], [m31image], context='2d')[0] # uvdist = numpy.sqrt(vt.data['uvw'][:, 0] ** 2 + vt.data['uvw'][:, 1] ** 2) # # model = create_image_from_visibility(vt, cellsize=cellsize, npixel=512) # dirty, sumwt = invert_list_serial_workflow([vt], [model], context='2d')[0] # psf, sumwt = invert_list_serial_workflow([vt], [model], context='2d', dopsf=True)[0] # # show_image(dirty) # print("Max, min in dirty image = %.6f, %.6f, sumwt = %f" % (dirty.data.max(), dirty.data.min(), sumwt)) # # print("Max, min in PSF = %.6f, %.6f, sumwt = %f" % (psf.data.max(), psf.data.min(), sumwt)) # results_dir="/Users/f.wang" # export_image_to_fits(dirty, '%s/imaging_dirty.fits' % (results_dir)) # export_image_to_fits(psf, '%s/imaging_psf.fits' % (results_dir)) v = convert_visibility_to_blockvisibility(vt) vis_list=[] vis_list.append(v) export_blockvisility_to_ms(msoutfile, vis_list,source_name='M31')