def test_mpccal_ICAL_onesource(self): self.actualSetup(nsources=1, nvoronoi=1) model = create_empty_image_like(self.theta_list[0].image) if rsexecute.using_dask: progress = None else: progress = self.progress future_vis = rsexecute.scatter(self.all_skymodel_noniso_vis) future_model = rsexecute.scatter(model) future_theta_list = rsexecute.scatter(self.theta_list) result = mpccal_skymodel_list_rsexecute_workflow(future_vis, future_model, future_theta_list, mpccal_progress=progress, nmajor=5, context='2d', algorithm='hogbom', scales=[0, 3, 10], fractional_threshold=0.15, threshold=0.05, gain=0.1, niter=1000, psf_support=256, deconvolve_facets=8, deconvolve_overlap=16, deconvolve_taper='tukey') (self.theta_list, residual) = rsexecute.compute(result, sync=True) combined_model = calculate_skymodel_equivalent_image(self.theta_list) psf_obs = invert_list_rsexecute_workflow([self.all_skymodel_noniso_vis], [model], context='2d', dopsf=True) result = restore_list_rsexecute_workflow([combined_model], psf_obs, [(residual, 0.0)]) result = rsexecute.compute(result, sync=True) if self.persist: export_image_to_fits(residual, rascil_path('test_results/test_mpccal_ical_onesource_residual.fits')) if self.persist: export_image_to_fits(result[0], rascil_path('test_results/test_mpccal_ical_onesource_restored.fits')) if self.persist: export_image_to_fits(combined_model, rascil_path('test_results/test_mpccal_ical_onesource_deconvolved.fits')) recovered_mpccal_components = find_skycomponents(result[0], fwhm=2, threshold=0.32, npixels=12) def max_flux(elem): return numpy.max(elem.flux) recovered_mpccal_components = sorted(recovered_mpccal_components, key=max_flux, reverse=True) assert recovered_mpccal_components[0].name == 'Segment 0', recovered_mpccal_components[0].name assert numpy.abs(recovered_mpccal_components[0].flux[0, 0] - 1.138095494391862) < 1e-6, \ recovered_mpccal_components[0].flux[0, 0] newscreen = create_empty_image_like(self.screen) gaintables = [th.gaintable for th in self.theta_list] newscreen, weights = grid_gaintable_to_screen(self.all_skymodel_noniso_blockvis, gaintables, newscreen) if self.persist: export_image_to_fits(newscreen, rascil_path('test_results/test_mpccal_ical_onesource_screen.fits')) if self.persist: export_image_to_fits(weights, rascil_path('test_results/test_mpccal_ical_onesource_screenweights.fits')) rsexecute.close()
def progress(res, tl_list, gt_list, it, context='MPCCAL'): print('Iteration %d' % it) print( qa_image(res, context='%s residual image: iteration %d' % (context, it))) export_image_to_fits( res, rascil_path( "test_results/low-sims-mpc-%s-residual_iteration%d_rmax%.1f.fits" % (context, it, rmax))) show_image(res, title='%s residual image: iteration %d' % (context, it)) plt.show(block=block_plots) combined_model = calculate_skymodel_equivalent_image(tl_list) print( qa_image(combined_model, context='Combined model: iteration %d' % it)) export_image_to_fits( combined_model, rascil_path( "test_results/low-sims-mpc-%s-model_iteration%d_rmax%.1f.fits" % (context, it, rmax))) plt.clf() for i in range(len(tl_list)): plt.plot( numpy.angle(tl_list[i].gaintable.gain[:, :, 0, 0, 0]).flatten(), numpy.angle(gt_list[i]['T'].gain[:, :, 0, 0, 0]).flatten(), '.') plt.xlabel('Current phase') plt.ylabel('Update to phase') plt.title("%s iteration%d: Change in phase" % (context, it)) plt.savefig( rascil_path( "test_results/low-sims-mpc-%s-skymodel-phase-change_iteration%d.jpg" % (context, it))) plt.show(block=block_plots) return tl_list
def test_create_list(self): if not self.casacore_available: return msfile = rascil_path("data/vis/xcasa.ms") self.vis = create_blockvisibility_from_ms(msfile) for v in self.vis: assert v.vis.data.shape[-1] == 4 assert v.polarisation_frame.type == "circular"
def test_create_list_spectral(self): if not self.casacore_available: return msfile = rascil_path("data/vis/ASKAP_example.ms") vis_by_channel = list() nchan_ave = 16 nchan = 192 for schan in range(0, nchan, nchan_ave): max_chan = min(nchan, schan + nchan_ave) v = create_visibility_from_ms(msfile, range(schan, max_chan)) vis_by_channel.append(v[0]) assert len(vis_by_channel) == 12 for v in vis_by_channel: assert v.vis.data.shape[-1] == 4 assert v.polarisation_frame.type == "linear"
def test_create_list_slice_visibility(self): if not self.casacore_available: return msfile = rascil_path("data/vis/ASKAP_example.ms") vis_by_channel = list() nchan_ave = 16 nchan = 192 for schan in range(0, nchan, nchan_ave): max_chan = min(nchan, schan + nchan_ave) v = create_visibility_from_ms(msfile, start_chan=schan, end_chan=max_chan - 1) nchannels = len(numpy.unique(v[0].frequency)) assert nchannels == nchan_ave vis_by_channel.append(v[0]) assert len(vis_by_channel) == 12 for v in vis_by_channel: assert v.vis.data.shape[-1] == 4 assert v.polarisation_frame.type == "linear" assert numpy.max(numpy.abs(v.vis)) > 0.0 assert numpy.max(numpy.abs(v.flagged_vis)) > 0.0 assert numpy.sum(v.weight) > 0.0 assert numpy.sum(v.flagged_weight) > 0.0
default='wstack', help='Context: 2d|timeslice|wstack') parser.add_argument('--nchan', type=int, default=40, help='Number of channels to process') parser.add_argument('--scheduler', type=str, default=None, help='Dask scheduler') args = parser.parse_args() print(args) # Put the results in the test_results directory results_dir = rascil_path('test_results') dask_dir = rascil_path('test_results/dask-work-space') # Since the processing is distributed over multiple processes we have to tell each Dask worker # where to send the log messages def init_logging(): logging.basicConfig( filename='%s/dprepb-pipeline.log' % results_dir, filemode='a', format='%(asctime)s,%(msecs)d %(name)s %(levelname)s %(message)s', datefmt='%H:%M:%S', level=logging.INFO) log = logging.getLogger() logging.info("Starting Imaging pipeline")
phasecentre=phasecentre, frequency=frequency, polarisation_frame=PolarisationFrame('stokesI'), radius=0.15) all_components = apply_beam_to_skycomponent(original_gleam_components, beam) all_components = filter_skycomponents_by_flux(all_components, flux_min=flux_limit) all_components = sorted(all_components, key=lambda comp: numpy.max(comp.flux), reverse=True) print("Number of components in simulation %d" % len(all_components)) screen = import_image_from_fits( rascil_path('data/models/test_mpc_screen.fits')) all_gaintables = create_gaintable_from_screen(block_vis, all_components, screen) all_skymodel = [ SkyModel(components=[all_components[i]], gaintable=all_gaintables[i]) for i, sm in enumerate(all_components) ] ####################################################################################################### # Calculate visibility by using the predict_skymodel function which applies a different gaintable table # for each skymodel. We do the calculation in chunks of nworkers skymodels. all_skymodel_blockvis = copy_visibility(block_vis, zero=True) all_skymodel_vis = convert_blockvisibility_to_visibility( all_skymodel_blockvis)
""" Simulation of observation for subsequent processing """ from rascil.data_models import rascil_path results_dir = rascil_path('test_results') dask_dir = rascil_path('test_results/dask-work-space') import numpy import logging from astropy.coordinates import SkyCoord from astropy import units as u from rascil.data_models import SkyModel, PolarisationFrame, export_skymodel_to_hdf5, \ export_blockvisibility_to_hdf5 from rascil.processing_components.image.operations import create_empty_image_like from rascil.processing_components import create_low_test_image_from_gleam, advise_wide_field, \ convert_blockvisibility_to_visibility, convert_visibility_to_blockvisibility from rascil.workflows import predict_list_rsexecute_workflow, simulate_list_rsexecute_workflow, \ corrupt_list_rsexecute_workflow from rascil.workflows.rsexecute.execution_support.rsexecute import rsexecute def init_logging(): logging.basicConfig( filename='%s/ska-pipeline.log' % results_dir, filemode='a',
from rascil.processing_components import create_blockvisibility_from_ms, export_image_to_fits, qa_image,\ deconvolve_cube, restore_cube, create_image_from_visibility, convert_blockvisibility_to_visibility,\ convert_visibility_to_stokes from rascil.workflows import invert_list_serial_workflow import logging log = logging.getLogger(__name__) log.setLevel(logging.DEBUG) log.addHandler(logging.StreamHandler(sys.stdout)) log.addHandler(logging.StreamHandler(sys.stderr)) if __name__ == '__main__': results_dir = rascil_path('test_results') bvt = create_blockvisibility_from_ms(rascil_path('data/vis/sim-2.ms'), start_chan=35, end_chan=39)[0] bvt.configuration.diameter[...] = 35.0 vt = convert_blockvisibility_to_visibility(bvt) vt = convert_visibility_to_stokes(vt) a2r = numpy.pi / (180.0 * 3600.0) model = create_image_from_visibility(vt, cellsize=20.0 * a2r, npixel=512, polarisation_frame=PolarisationFrame('stokesIQUV')) dirty, sumwt = invert_list_serial_workflow([vt], [model], context='2d')[0] psf, sumwt = invert_list_serial_workflow([vt], [model], context='2d', dopsf=True)[0] export_image_to_fits(dirty, '%s/rascil_imaging_sim_2_dirty.fits' % (results_dir)) export_image_to_fits(psf, '%s/rascil_imaging_sim_2_psf.fits' % (results_dir))
from rascil.processing_components import create_blockvisibility_from_ms, export_image_to_fits, qa_image,\ deconvolve_cube, restore_cube, create_image_from_visibility, convert_blockvisibility_to_visibility,\ convert_visibility_to_stokes from rascil.workflows import invert_list_serial_workflow import logging log = logging.getLogger(__name__) log.setLevel(logging.DEBUG) log.addHandler(logging.StreamHandler(sys.stdout)) log.addHandler(logging.StreamHandler(sys.stderr)) if __name__ == '__main__': results_dir = rascil_path('test_results') bvt = create_blockvisibility_from_ms(rascil_path('data/vis/sim-2.ms'), start_chan=35, end_chan=39)[0] bvt.configuration.diameter[...] = 35.0 vt = convert_blockvisibility_to_visibility(bvt) vt = convert_visibility_to_stokes(vt) a2r = numpy.pi / (180.0 * 3600.0) model = create_image_from_visibility( vt, cellsize=20.0 * a2r, npixel=512, polarisation_frame=PolarisationFrame('stokesIQUV'))
def actualSetup(self, nsources=None, nvoronoi=None): n_workers = 8 # Set up the observation: 10 minutes at transit, with 10s integration. # Skip 5/6 points to avoid outstation redundancy nfreqwin = 1 ntimes = 3 self.rmax = 2500.0 dec = -40.0 * u.deg frequency = [1e8] channel_bandwidth = [0.1e8] times = numpy.linspace(-10.0, 10.0, ntimes) * numpy.pi / (3600.0 * 12.0) phasecentre = SkyCoord(ra=+0.0 * u.deg, dec=dec, frame='icrs', equinox='J2000') low = create_named_configuration('LOWBD2', rmax=self.rmax) centre = numpy.mean(low.xyz, axis=0) distance = numpy.hypot(low.xyz[:, 0] - centre[0], low.xyz[:, 1] - centre[1], low.xyz[:, 2] - centre[2]) lowouter = low.data[distance > 1000.0][::6] lowcore = low.data[distance < 1000.0][::3] low.data = numpy.hstack((lowcore, lowouter)) blockvis = create_blockvisibility( low, times, frequency=frequency, channel_bandwidth=channel_bandwidth, weight=1.0, phasecentre=phasecentre, polarisation_frame=PolarisationFrame("stokesI"), zerow=True) vis = convert_blockvisibility_to_visibility(blockvis) advice = advise_wide_field(vis, guard_band_image=2.0, delA=0.02) cellsize = advice['cellsize'] npixel = advice['npixels2'] small_model = create_image_from_visibility(blockvis, npixel=512, frequency=frequency, nchan=nfreqwin, cellsize=cellsize, phasecentre=phasecentre) vis.data['imaging_weight'][...] = vis.data['weight'][...] vis = weight_list_serial_workflow([vis], [small_model])[0] vis = taper_list_serial_workflow([vis], 3 * cellsize)[0] blockvis = convert_visibility_to_blockvisibility(vis) # ### Generate the model from the GLEAM catalog, including application of the primary beam. beam = create_image_from_visibility(blockvis, npixel=npixel, frequency=frequency, nchan=nfreqwin, cellsize=cellsize, phasecentre=phasecentre) beam = create_low_test_beam(beam, use_local=False) flux_limit = 0.5 original_gleam_components = create_low_test_skycomponents_from_gleam( flux_limit=flux_limit, phasecentre=phasecentre, frequency=frequency, polarisation_frame=PolarisationFrame('stokesI'), radius=0.15) all_components = apply_beam_to_skycomponent(original_gleam_components, beam) all_components = filter_skycomponents_by_flux(all_components, flux_min=flux_limit) voronoi_components = filter_skycomponents_by_flux(all_components, flux_min=1.5) def max_flux(elem): return numpy.max(elem.flux) voronoi_components = sorted(voronoi_components, key=max_flux, reverse=True) if nsources is not None: all_components = [all_components[0]] if nvoronoi is not None: voronoi_components = [voronoi_components[0]] self.screen = import_image_from_fits( rascil_path('data/models/test_mpc_screen.fits')) all_gaintables = create_gaintable_from_screen(blockvis, all_components, self.screen) gleam_skymodel_noniso = [ SkyModel(components=[all_components[i]], gaintable=all_gaintables[i]) for i, sm in enumerate(all_components) ] # ### Now predict the visibility for each skymodel and apply the gaintable for that skymodel, # returning a list of visibilities, one for each skymodel. We then sum these to obtain # the total predicted visibility. All images and skycomponents in the same skymodel # get the same gaintable applied which means that in this case each skycomponent has a separate gaintable. self.all_skymodel_noniso_vis = convert_blockvisibility_to_visibility( blockvis) ngroup = n_workers future_vis = rsexecute.scatter(self.all_skymodel_noniso_vis) chunks = [ gleam_skymodel_noniso[i:i + ngroup] for i in range(0, len(gleam_skymodel_noniso), ngroup) ] for chunk in chunks: result = predict_skymodel_list_rsexecute_workflow(future_vis, chunk, context='2d', docal=True) work_vis = rsexecute.compute(result, sync=True) for w in work_vis: self.all_skymodel_noniso_vis.data['vis'] += w.data['vis'] assert numpy.max( numpy.abs(self.all_skymodel_noniso_vis.data['vis'])) > 0.0 self.all_skymodel_noniso_blockvis = convert_visibility_to_blockvisibility( self.all_skymodel_noniso_vis) # ### Remove weaker of components that are too close (0.02 rad) idx, voronoi_components = remove_neighbouring_components( voronoi_components, 0.02) model = create_image_from_visibility(blockvis, npixel=npixel, frequency=frequency, nchan=nfreqwin, cellsize=cellsize, phasecentre=phasecentre) # Use the gaintable for the brightest component as the starting gaintable all_gaintables[0].gain[...] = numpy.conjugate( all_gaintables[0].gain[...]) all_gaintables[0].gain[...] = 1.0 + 0.0j self.theta_list = initialize_skymodel_voronoi(model, voronoi_components, all_gaintables[0])
from astropy.coordinates import SkyCoord from astropy import units as u from rascil.data_models import PolarisationFrame, rascil_path from rascil.processing_components import create_visibility, export_image_to_fits, \ deconvolve_cube, restore_cube, create_named_configuration, create_test_image,\ create_image_from_visibility, advise_wide_field, invert_2d, predict_2d import logging log = logging.getLogger() log.setLevel(logging.DEBUG) log.addHandler(logging.StreamHandler(sys.stdout)) results_dir = rascil_path('test_results') # Construct LOW core configuration lowr3 = create_named_configuration('LOWBD2', rmax=750.0) # We create the visibility. This just makes the uvw, time, antenna1, antenna2, # weight columns in a table. We subsequently fill the visibility value in by # a predict step. times = numpy.zeros([1]) frequency = numpy.array([1e8]) channel_bandwidth = numpy.array([1e6]) phasecentre = SkyCoord(ra=+15.0 * u.deg, dec=-45.0 * u.deg, frame='icrs', equinox='J2000')
return v imaging_context = "ng" vis_slices = 1 print('Using {}'.format(imaging_context)) predicted_vislist = predict_list_rsexecute_workflow( vis_list, dprepb_model, context=imaging_context, vis_slices=vis_slices, verbosity=2) corrupted_vislist = corrupt_list_rsexecute_workflow(predicted_vislist, phase_error=1.0, seed=180555) corrupted_vislist = [ rsexecute.execute(print_max)(v) for v in corrupted_vislist ] export_list = [ rsexecute.execute(export_blockvisibility_to_ms)(rascil_path( '%s/ska-pipeline_simulation_vislist_%d.ms' % (results_dir, v)), [corrupted_vislist[v]]) for v, _ in enumerate(corrupted_vislist) ] print( 'About to run predict and corrupt to get corrupted visibility, and write files' ) rsexecute.compute(export_list, sync=True) rsexecute.close()