def test_ical_pipeline_global(self): self.actualSetUp(add_errors=True) controls = create_calibration_controls() controls['T']['first_selfcal'] = 1 controls['T']['timescale'] = 'auto' clean, residual, restored, gt_list = \ ical_list_serial_workflow(self.vis_list, model_imagelist=self.model_imagelist, context='2d', algorithm='mmclean', facets=1, scales=[0, 3, 10], niter=1000, fractional_threshold=0.1, threshold=0.1, nmoment=3, nmajor=5, gain=0.1, deconvolve_facets=4, deconvolve_overlap=32, deconvolve_taper='tukey', psf_support=64, calibration_context='T', controls=controls, do_selfcal=True, global_solution=True) centre = len(clean) // 2 if self.persist: export_image_to_fits(clean[centre], '%s/test_pipelines_ical_global_pipeline_serial_clean.fits' % self.dir) export_image_to_fits(residual[centre][0], '%s/test_pipelines_ical_global_pipeline_serial_residual.fits' % self.dir) export_image_to_fits(restored[centre], '%s/test_pipelines_ical_global_pipeline_serial_restored.fits' % self.dir) export_gaintable_to_hdf5(gt_list[0]['T'], '%s/test_pipelines_ical_global_pipeline_serial_gaintable.hdf5' % self.dir) qa = qa_image(restored[centre]) assert numpy.abs(qa.data['max'] - 98.92656340122159) < 1.0, str(qa) assert numpy.abs(qa.data['min'] + 0.7024492707920869) < 1.0, str(qa)
def test_continuum_imaging_pipeline(self): self.actualSetUp(add_errors=False, zerow=True) clean, residual, restored = \ continuum_imaging_list_serial_workflow(self.vis_list, model_imagelist=self.model_imagelist, context='2d', algorithm='mmclean', facets=1, scales=[0, 3, 10], niter=1000, fractional_threshold=0.1, threshold=0.1, nmoment=3, nmajor=5, gain=0.1, deconvolve_facets=4, deconvolve_overlap=32, deconvolve_taper='tukey', psf_support=64) centre = len(clean) // 2 if self.persist: export_image_to_fits( clean[centre], '%s/test_pipelines_continuum_imaging_pipeline_serial_clean.fits' % self.dir) export_image_to_fits( residual[centre][0], '%s/test_pipelines_continuum_imaging_pipeline_serial_residual.fits' % self.dir) export_image_to_fits( restored[centre], '%s/test_pipelines_continuum_imaging_pipeline_serial_restored.fits' % self.dir) qa = qa_image(restored[centre]) assert numpy.abs(qa.data['max'] - 100.13762476849081) < 1.0, str(qa) assert numpy.abs(qa.data['min'] + 0.03627273884170454) < 1.0, str(qa)
def test_residual_list(self): self.actualSetUp(zerow=True) centre = self.freqwin // 2 residual_image_list = residual_list_serial_workflow(self.vis_list, self.model_list, context='2d') qa = qa_image(residual_image_list[centre][0]) assert numpy.abs(qa.data['max'] - 0.35139716991480785) < 1.0, str(qa) assert numpy.abs(qa.data['min'] + 0.7681701460717593) < 1.0, str(qa)
def progress(res, tl_list, gt_list, it, context='MPCCAL'): print('Iteration %d' % it) print( qa_image(res, context='%s residual image: iteration %d' % (context, it))) export_image_to_fits( res, arl_path( "test_results/low-sims-mpc-%s-residual_iteration%d_rmax%.1f.fits" % (context, it, rmax))) show_image(res, title='%s residual image: iteration %d' % (context, it)) plt.show(block=block_plots) combined_model = calculate_skymodel_equivalent_image(tl_list) print( qa_image(combined_model, context='Combined model: iteration %d' % it)) export_image_to_fits( combined_model, arl_path( "test_results/low-sims-mpc-%s-model_iteration%d_rmax%.1f.fits" % (context, it, rmax))) plt.clf() for i in range(len(tl_list)): plt.plot( numpy.angle(tl_list[i].gaintable.gain[:, :, 0, 0, 0]).flatten(), numpy.angle(gt_list[i]['T'].gain[:, :, 0, 0, 0]).flatten(), '.') plt.xlabel('Current phase') plt.ylabel('Update to phase') plt.title("%s iteration%d: Change in phase" % (context, it)) plt.savefig( arl_path( "test_results/low-sims-mpc-%s-skymodel-phase-change_iteration%d.jpg" % (context, it))) plt.show(block=block_plots) return tl_list
def test_random(self): error_dirty, sumwt = self.simulation(self.get_args(), '') qa = qa_image(error_dirty) numpy.testing.assert_almost_equal(qa.data['max'], 2.2055849698035616e-06, 12) numpy.testing.assert_almost_equal(qa.data['min'], -6.838117387793031e-07, 12) numpy.testing.assert_almost_equal(qa.data['rms'], 3.7224203394509413e-07, 12)
def test_wind(self): error_dirty, sumwt = self.simulation(self.get_args(), 'wind') qa = qa_image(error_dirty) numpy.testing.assert_almost_equal(qa.data['max'], 1.2440965953105578e-06, 12) numpy.testing.assert_almost_equal(qa.data['min'], -3.836051633637655e-06, 12) numpy.testing.assert_almost_equal(qa.data['rms'], 5.840397284050296e-07, 12)
def test_gravity(self): if os.path.isdir(arl_path('models/interpolated')): error_dirty, sumwt = self.simulation(self.get_args(), 'gravity') qa = qa_image(error_dirty) numpy.testing.assert_almost_equal(qa.data['max'], 2.2055849698035616e-06, 12) numpy.testing.assert_almost_equal(qa.data['min'], -6.838117387793031e-07, 12) numpy.testing.assert_almost_equal(qa.data['rms'], 3.7224203394509413e-07, 12)
def test_ical_pipeline(self): amp_errors = {'T': 0.0, 'G': 0.00, 'B': 0.0} phase_errors = {'T': 0.1, 'G': 0.0, 'B': 0.0} self.actualSetUp(add_errors=True, block=True, amp_errors=amp_errors, phase_errors=phase_errors) controls = create_calibration_controls() controls['T']['first_selfcal'] = 1 controls['G']['first_selfcal'] = 3 controls['B']['first_selfcal'] = 4 controls['T']['timescale'] = 'auto' controls['G']['timescale'] = 'auto' controls['B']['timescale'] = 1e5 clean, residual, restored = \ ical_list_serial_workflow(self.vis_list, model_imagelist=self.model_imagelist, context='2d', calibration_context='T', controls=controls, do_selfcal=True, global_solution=False, algorithm='mmclean', facets=1, scales=[0, 3, 10], niter=1000, fractional_threshold=0.1, nmoments=2, nchan=self.freqwin, threshold=2.0, nmajor=5, gain=0.1, deconvolve_facets=8, deconvolve_overlap=16, deconvolve_taper='tukey') centre = len(clean) // 2 export_image_to_fits( clean[centre], '%s/test_pipelines_ical_pipeline_clean.fits' % self.dir) export_image_to_fits( residual[centre][0], '%s/test_pipelines_ical_pipeline_residual.fits' % self.dir) export_image_to_fits( restored[centre], '%s/test_pipelines_ical_pipeline_restored.fits' % self.dir) qa = qa_image(restored[centre]) assert numpy.abs(qa.data['max'] - 100.13739440876233) < 1.0, str(qa) assert numpy.abs(qa.data['min'] + 0.03644435471804354) < 1.0, str(qa)
elif context == 'wstack': future = invert_list_arlexecute_workflow(vt_list, targetimage_list, context=context, vis_slices=31) else: future = invert_list_arlexecute_workflow(vt_list, targetimage_list, context=context) result = arlexecute.compute(future, sync=True) targetimage = result[0][0] show_image(targetimage) plt.title(context) plt.savefig('2_'+context+'.jpg') #plt.show() print("Dirty Image %s" % qa_image(targetimage, context="imaging-fits notebook, using processor %s" % context)) export_image_to_fits(targetimage, '%s/imaging-fits_dirty_%s.fits' % (results_dir, context)) comps = find_skycomponents(targetimage, fwhm=1.0, threshold=10.0, npixels=5) plt.clf() for comp in comps: distance = comp.direction.separation(model.phasecentre) dft_flux = sum_visibility(vt, comp.direction)[0] err = (comp.flux[0, 0] - dft_flux) / dft_flux plt.plot(distance, err, '.', color='r') plt.ylabel('Fractional error of image vs DFT') plt.xlabel('Distance from phasecentre (deg)') plt.title( "Fractional error in %s recovered flux vs distance from phasecentre" % context)
# Create and execute graphs to make the dirty image and PSF # In[ ]: end = time.time() log.info('%d: invert finished' % (rank)) print('%d: invert finished in %f seconds' % (rank, end - start), flush=True) if rank == 0: #print("sumwts",flush=True) #print(dirty_list[0][1]) log.info('After invert to get dirty image') dirty = dirty_list[0][0] #show_image(dirty, cm='Greys', vmax=1.0, vmin=-0.1) #plt.show() print(qa_image(dirty)) export_image_to_fits(dirty, '%s/imaging-dirty.fits' % (results_dir)) log.info('After invert to get PSF') psf = psf_list[0][0] #show_image(psf, cm='Greys', vmax=0.1, vmin=-0.01) #plt.show() print(qa_image(psf)) export_image_to_fits(psf, '%s/imaging-psf.fits' % (results_dir)) # Now deconvolve using msclean # In[ ]: log.info('%d: About to run deconvolve' % (rank)) print('%d: About to run deconvolve' % (rank), flush=True)
else: d, sumwt = invert_list_serial_workflow([v], [m], context=context, dopsf=False, vis_slices=vis_slices)[0] p, sumwt = invert_list_serial_workflow([v], [m], context=context, dopsf=True, vis_slices=vis_slices)[0] c, resid = deconvolve_cube(d, p, m, threshold=0.01, fracthresh=0.01, window_shape='quarter', niter=100, gain=0.1, algorithm='hogbom-complex') r = restore_cube(c, p, resid, psfwidth=psfwidth) return r print('About assemble cubes and deconvolve each frequency') restored_list = [load_invert_and_deconvolve(c) for c in range(nchan)] restored_cube = image_gather_channels(restored_list) print("Processing took %.3f s" % (time.time() - start)) print(qa_image(restored_cube, context='CLEAN restored cube')) export_image_to_fits( restored_cube, '%s/dprepb_serial_%s_clean_restored_cube.fits' % (results_dir, context))
progress, context='ICAL'), nmajor=args.ical_nmajor, context='2d', algorithm='hogbom', fractional_threshold=0.3, threshold=0.1, gain=0.1, niter=1000, psf_support=512, deconvolve_facets=8, deconvolve_overlap=16, deconvolve_taper='tukey') (ical_skymodel, residual) = arlexecute.compute(result, sync=True) print(qa_image(residual, context='ICAL residual image')) print('ical finished') combined_model = calculate_skymodel_equivalent_image(ical_skymodel) print(qa_image(combined_model, context='ICAL combined model')) psf_obs = invert_list_arlexecute_workflow([future_vis], [future_model], context='2d', dopsf=True) result = restore_list_arlexecute_workflow([combined_model], psf_obs, [(residual, 0.0)]) result = arlexecute.compute(result, sync=True) ical_restored = result[0] export_image_to_fits( ical_restored,
chunks = [all_skymodel[i:i + ngroup] for i in range(0, len(all_skymodel), ngroup)] for chunk in chunks: result = predict_skymodel_list_arlexecute_workflow(future_vis, chunk, context='2d', docal=True) work_vis = arlexecute.compute(result, sync=True) for w in work_vis: all_skymodel_vis.data['vis'] += w.data['vis'] assert numpy.max(numpy.abs(all_skymodel_vis.data['vis'])) > 0.0 all_skymodel_blockvis = convert_visibility_to_blockvisibility(all_skymodel_vis) ####################################################################################################### # Now proceed to run MPCCAL in ICAL mode i.e. with only one skymodel def progress(res, tl_list, gt_list, it, context='MPCCAL'): print('Iteration %d' % it) print(qa_image(res, context='%s residual image: iteration %d' % (context, it))) export_image_to_fits(res, arl_path("test_results/low-sims-mpc-%s-residual_iteration%d_rmax%.1f.fits" % (context, it, rmax))) show_image(res, title='%s residual image: iteration %d' % (context, it)) plt.show(block=block_plots) combined_model = calculate_skymodel_equivalent_image(tl_list) print(qa_image(combined_model, context='Combined model: iteration %d' % it)) export_image_to_fits(combined_model, arl_path("test_results/low-sims-mpc-%s-model_iteration%d_rmax%.1f.fits" % (context, it, rmax))) plt.clf() for i in range(len(tl_list)): plt.plot(numpy.angle(tl_list[i].gaintable.gain[:, :, 0, 0, 0]).flatten(), numpy.angle(gt_list[i]['T'].gain[:, :, 0, 0, 0]).flatten(), '.')
context='wstack', vis_slices=vis_slices, dopsf=False) psf_list = invert_list_mpi_workflow(predicted_vislist, model_list, context='wstack', vis_slices=vis_slices, dopsf=True) # Create and execute graphs to make the dirty image and PSF # In[ ]: if rank == 0: print("sumwts", flush=True) print(dirty_list[0][1]) log.info('About to run invert to get dirty image') dirty = dirty_list[0][0] #show_image(dirty, cm='Greys', vmax=1.0, vmin=-0.1) #plt.show() print(qa_image(dirty)) export_image_to_fits(dirty, '%s/imaging-dirty.fits' % (results_dir)) log.info('About to run invert to get PSF') psf = psf_list[0][0] #show_image(psf, cm='Greys', vmax=0.1, vmin=-0.01) #plt.show() print(qa_image(psf)) export_image_to_fits(psf, '%s/imaging-psf.fits' % (results_dir))
vt = convert_blockvisibility_to_visibility(bvt) vt = convert_visibility_to_stokes(vt) cellsize = 20.0 * numpy.pi / (180.0 * 3600.0) npixel = 512 model = create_image_from_visibility(vt, cellsize=cellsize, npixel=npixel, polarisation_frame=PolarisationFrame('stokesIQUV')) dirty, sumwt = invert_list_serial_workflow([vt], [model], context='2d')[0] psf, sumwt = invert_list_serial_workflow([vt], [model], context='2d', dopsf=True)[0] export_image_to_fits(dirty, '%s/compare_imaging_sim2_dirty.fits' % (results_dir)) export_image_to_fits(psf, '%s/compare_imaging_sim2_psf.fits' % (results_dir)) # Deconvolve using clean comp, residual = deconvolve_cube(dirty, psf, niter=10000, threshold=0.001, fractional_threshold=0.001, window_shape='quarter', gain=0.7, scales=[0, 3, 10, 30]) restored = restore_cube(comp, psf, residual) export_image_to_fits(restored, '%s/compare_imaging_sim2_restored.fits' % (results_dir)) export_image_to_fits(residual, '%s/compare_imaging_sim2_residual.fits' % (results_dir)) qa = qa_image(restored) assert numpy.abs(qa.data['max'] - 1.006140596404203) < 1e-7, qa assert numpy.abs(qa.data['maxabs'] - 1.006140596404203) < 1e-7, qa assert numpy.abs(qa.data['min'] + 0.23890808520954754) < 1e-7, qa assert numpy.abs(qa.data['rms'] - 0.007366519782047875) < 1e-7, qa assert numpy.abs(qa.data['medianabs'] - 0.0005590537883509844) < 1e-7, qa except ModuleNotFoundError as error: print(error.__class__.__name__ + ": " + error.msg) pass