def test_ical_pipeline(self): amp_errors = {'T': 0.0, 'G': 0.00, 'B': 0.0} phase_errors = {'T': 0.1, 'G': 0.0, 'B': 0.0} self.actualSetUp(add_errors=True, block=True, amp_errors=amp_errors, phase_errors=phase_errors) controls = create_calibration_controls() controls['T']['first_selfcal'] = 1 controls['G']['first_selfcal'] = 3 controls['B']['first_selfcal'] = 4 controls['T']['timescale'] = 'auto' controls['G']['timescale'] = 'auto' controls['B']['timescale'] = 1e5 ical_list = \ ical_workflow(self.vis_list, model_imagelist=self.model_imagelist, context='2d', calibration_context='T', controls=controls, do_selfcal=True, global_solution=False, algorithm='mmclean', facets=1, scales=[0, 3, 10], niter=1000, fractional_threshold=0.1, nmoments=2, nchan=self.freqwin, threshold=2.0, nmajor=5, gain=0.1, deconvolve_facets=8, deconvolve_overlap=16, deconvolve_taper='tukey') clean, residual, restored = arlexecute.compute(ical_list, sync=True) export_image_to_fits(clean[0], '%s/test_pipelines_ical_pipeline_clean.fits' % self.dir) export_image_to_fits(residual[0][0], '%s/test_pipelines_ical_pipeline_residual.fits' % self.dir) export_image_to_fits(restored[0], '%s/test_pipelines_ical_pipeline_restored.fits' % self.dir) qa = qa_image(restored[0]) assert numpy.abs(qa.data['max'] - 116.9) < 1.0, str(qa) assert numpy.abs(qa.data['min'] + 0.118) < 1.0, str(qa)
def insert_unittest_errors(vt, seed=180555, amp_errors=None, phase_errors=None): """Simulate gain errors and apply :param vt: :param seed: Random number seed, set to big integer repeat values from run to run :param phase_errors: e.g. {'T': 1.0, 'G': 0.1, 'B': 0.01} :param amp_errors: e.g. {'T': 0.0, 'G': 0.01, 'B': 0.01} :return: """ numpy.random.seed(seed) controls = create_calibration_controls() if amp_errors is None: amp_errors = {'T': 0.0, 'G': 0.01, 'B': 0.01} if phase_errors is None: phase_errors = {'T': 1.0, 'G': 0.1, 'B': 0.01} for c in "TGB": gaintable = \ create_gaintable_from_blockvisibility(vt, timeslice=controls[c]['timeslice']) gaintable = simulate_gaintable(gaintable, timeslice=controls[c]['timeslice'], phase_only=controls[c]['phase_only'], crosspol=controls[c]['shape'] == 'matrix', phase_error=phase_errors[c], amplitude_error=amp_errors[c]) vt = apply_gaintable(vt, gaintable, inverse=True, timeslice=controls[c]['timeslice']) return vt
def test_calibrate_T_function(self): self.actualSetup('stokesI', 'stokesI', f=[100.0]) # Prepare the corrupted visibility data_models gt = create_gaintable_from_blockvisibility(self.vis) log.info("Created gain table: %s" % (gaintable_summary(gt))) gt = simulate_gaintable(gt, phase_error=10.0, amplitude_error=0.0) original = copy_visibility(self.vis) self.vis = apply_gaintable(self.vis, gt, vis_slices=None) # Now get the control dictionary and calibrate controls = create_calibration_controls() controls['T']['first_selfcal'] = 0 calibrated_vis, gaintables = calibrate_function( self.vis, original, calibration_context='T', controls=controls) residual = numpy.max(gaintables['T'].residual) assert residual < 1e-8, "Max T residual = %s" % (residual)
# Load data from previous simulation vislist = import_blockvisibility_from_hdf5('gleam_simulation_vislist.hdf') print(vislist[0]) cellsize = 0.001 npixel = 1024 pol_frame = PolarisationFrame("stokesI") model_list = [ arlexecute.execute(create_image_from_visibility)( v, npixel=1024, cellsize=cellsize, polarisation_frame=pol_frame) for v in vislist ] controls = create_calibration_controls() controls['T']['first_selfcal'] = 1 controls['G']['first_selfcal'] = 3 controls['B']['first_selfcal'] = 4 controls['T']['timescale'] = 'auto' controls['G']['timescale'] = 'auto' controls['B']['timescale'] = 1e5 pp.pprint(controls) # In[ ]: future_vislist = arlexecute.scatter(vislist) ntimes = len(vislist[0].time)
def main(): """Run the workflow.""" init_logging() LOG.info("Starting imaging-pipeline") # Read parameters PARFILE = 'parameters.json' if len(sys.argv) > 1: PARFILE = sys.argv[1] LOG.info("JSON parameter file = %s", PARFILE) try: with open(PARFILE, "r") as par_file: jspar = json.load(par_file) except AssertionError as error: LOG.critical('ERROR %s', error) return # We will use dask arlexecute.set_client(get_dask_Client()) arlexecute.run(init_logging) # Import visibility list from HDF5 file vis_list = import_blockvisibility_from_hdf5( '%s/%s' % (RESULTS_DIR, jspar["files"]["vis_list"])) # Now read the BlockVisibilities constructed using a model drawn from GLEAM predicted_vislist = import_blockvisibility_from_hdf5( '%s/%s' % (RESULTS_DIR, jspar["files"]["predicted_vis_list"])) corrupted_vislist = import_blockvisibility_from_hdf5( '%s/%s' % (RESULTS_DIR, jspar["files"]["corrupted_vis_list"])) # Reproduce parameters from the visibility data ntimes = vis_list[0].nvis phasecentre = vis_list[0].phasecentre print(phasecentre) polframe = vis_list[0].polarisation_frame.type LOG.info("Polarisation Frame of vis_list: %s", polframe) wprojection_planes = jspar["advice"]["wprojection_planes"] guard_band_image = jspar["advice"]["guard_band_image"] delA = jspar["advice"]["delA"] advice_low = advise_wide_field(vis_list[0], guard_band_image=guard_band_image, delA=delA, wprojection_planes=wprojection_planes) advice_high = advise_wide_field(vis_list[-1], guard_band_image=guard_band_image, delA=delA, wprojection_planes=wprojection_planes) vis_slices = advice_low['vis_slices'] npixel = advice_high['npixels2'] cellsize = min(advice_low['cellsize'], advice_high['cellsize']) # Recovering frequencies fstart = vis_list[0].frequency fend = vis_list[-1].frequency num_freq_win = len(vis_list) frequency = numpy.linspace(fstart, fend, num_freq_win) # Recovering bandwidths channel_bandwidth = numpy.array(num_freq_win * [vis_list[1].frequency - vis_list[0].frequency]) # Get the LSM. This is currently blank. model_list = [ arlexecute.execute(create_image_from_visibility)( vis_list[f], npixel=npixel, frequency=[frequency[f]], channel_bandwidth=[channel_bandwidth[f]], cellsize=cellsize, phasecentre=phasecentre, polarisation_frame=PolarisationFrame(polframe)) for f, freq in enumerate(frequency) ] # future_predicted_vislist = arlexecute.scatter(predicted_vislist) # Create and execute graphs to make the dirty image and PSF # LOG.info('About to run invert to get dirty image') # dirty_list = invert_component(future_predicted_vislist, model_list, # context='wstack', # vis_slices=vis_slices, dopsf=False) # dirty_list = arlexecute.compute(dirty_list, sync=True) # LOG.info('About to run invert to get PSF') # psf_list = invert_component(future_predicted_vislist, model_list, # context='wstack', # vis_slices=vis_slices, dopsf=True) # psf_list = arlexecute.compute(psf_list, sync=True) # Now deconvolve using msclean # LOG.info('About to run deconvolve') # deconvolve_list, _ = deconvolve_component( # dirty_list, psf_list, # model_imagelist=model_list, # deconvolve_facets=8, # deconvolve_overlap=16, # deconvolve_taper='tukey', # scales=[0, 3, 10], # algorithm='msclean', # niter=1000, # fractional_threshold=0.1, # threshold=0.1, # gain=0.1, # psf_support=64) # deconvolved = arlexecute.compute(deconvolve_list, sync=True) LOG.info('About to run continuum imaging') continuum_imaging_list = continuum_imaging_arlexecute( predicted_vislist, model_imagelist = model_list, context = jspar["processing"]["continuum_imaging"]["context"] , #'wstack', vis_slices = vis_slices, scales = jspar["processing"]["continuum_imaging"]["scales"], #[0, 3, 10], algorithm = jspar["processing"]["continuum_imaging"]["algorithm"], #'mmclean', nmoment = jspar["processing"]["continuum_imaging"]["nmoment"], #3, niter = jspar["processing"]["continuum_imaging"]["niter"], #1000, fractional_threshold = jspar["processing"]["continuum_imaging"]["fractional_threshold"], #0.1, threshold = jspar["processing"]["continuum_imaging"]["threshold"], #0.1, nmajor = jspar["processing"]["continuum_imaging"]["nmajor"], #5, gain = jspar["processing"]["continuum_imaging"]["gain"], #0.25, deconvolve_facets = jspar["processing"]["continuum_imaging"]["deconvolve_facets"], #8, deconvolve_overlap = jspar["processing"]["continuum_imaging"]["deconvolve_overlap"], #16, deconvolve_taper = jspar["processing"]["continuum_imaging"]["deconvolve_taper"], #'tukey', psf_support = jspar["processing"]["continuum_imaging"]["psf_support"] ) #64) result = arlexecute.compute(continuum_imaging_list, sync=True) deconvolved = result[0][0] residual = result[1][0] restored = result[2][0] print(qa_image(deconvolved, context='Clean image - no selfcal')) print(qa_image(restored, context='Restored clean image - no selfcal')) export_image_to_fits(restored, '%s/%s' % (RESULTS_DIR, jspar["files"]["continuum_imaging_restored"])) print(qa_image(residual[0], context='Residual clean image - no selfcal')) export_image_to_fits(residual[0], '%s/%s' % (RESULTS_DIR, jspar["files"]["continuum_imaging_residual"])) controls = create_calibration_controls() controls['T']['first_selfcal'] = jspar["processing"]["controls"]["T"]["first_selfcal"] controls['G']['first_selfcal'] = jspar["processing"]["controls"]["G"]["first_selfcal"] controls['B']['first_selfcal'] = jspar["processing"]["controls"]["B"]["first_selfcal"] controls['T']['timescale'] = jspar["processing"]["controls"]["T"]["timescale"] controls['G']['timescale'] = jspar["processing"]["controls"]["G"]["timescale"] controls['B']['timescale'] = jspar["processing"]["controls"]["B"]["timescale"] PP.pprint(controls) future_corrupted_vislist = arlexecute.scatter(corrupted_vislist) ical_list = ical_arlexecute(future_corrupted_vislist, model_imagelist = model_list, context = jspar["processing"]["ical"]["context"] , #'wstack', calibration_context = jspar["processing"]["ical"]["calibration_context"] , #'TG', controls = controls, vis_slices = ntimes, scales = jspar["processing"]["ical"]["scales"], #[0, 3, 10], timeslice = jspar["processing"]["ical"]["timeslice"], #'auto', algorithm = jspar["processing"]["ical"]["algorithm"], #'mmclean', nmoment = jspar["processing"]["ical"]["nmoment"], #3, niter = jspar["processing"]["ical"]["niter"], #1000, fractional_threshold = jspar["processing"]["ical"]["fractional_threshold"], #0.1, threshold = jspar["processing"]["ical"]["threshold"], #0.1, nmajor = jspar["processing"]["ical"]["nmajor"], #5, gain = jspar["processing"]["ical"]["gain"], #0.25, deconvolve_facets = jspar["processing"]["ical"]["deconvolve_facets"], #8, deconvolve_overlap = jspar["processing"]["ical"]["deconvolve_overlap"], #16, deconvolve_taper = jspar["processing"]["ical"]["deconvolve_taper"], #'tukey', global_solution = jspar["processing"]["ical"]["global_solution"], #False, do_selfcal = jspar["processing"]["ical"]["do_selfcal"], #True, psf_support = jspar["processing"]["ical"]["psf_support"]) #64 LOG.info('About to run ical') result = arlexecute.compute(ical_list, sync=True) deconvolved = result[0][0] residual = result[1][0] restored = result[2][0] print(qa_image(deconvolved, context='Clean image')) print(qa_image(restored, context='Restored clean image')) export_image_to_fits(restored, '%s/%s' % (RESULTS_DIR, jspar["files"]["ical_restored"])) print(qa_image(residual[0], context='Residual clean image')) export_image_to_fits(residual[0], '%s/%s' % (RESULTS_DIR, jspar["files"]["ical_residual"])) arlexecute.close()
def trial_case(results, seed=180555, context='wstack', nworkers=8, threads_per_worker=1, memory=8, processes=True, order='frequency', nfreqwin=7, ntimes=3, rmax=750.0, facets=1, wprojection_planes=1, use_dask=True, use_serial=False): """ Single trial for performance-timings Simulates visibilities from GLEAM including phase errors Makes dirty image and PSF Runs ICAL pipeline The results are in a dictionary: 'context': input - a string describing concisely the purpose of the test 'time overall', overall execution time (s) 'time create gleam', time to create GLEAM prediction graph 'time predict', time to execute GLEAM prediction graph 'time corrupt', time to corrupt data_models 'time invert', time to make dirty image 'time psf invert', time to make PSF 'time ICAL graph', time to create ICAL graph 'time ICAL', time to execute ICAL graph 'context', type of imaging e.g. 'wstack' 'nworkers', number of workers to create 'threads_per_worker', 'nnodes', Number of nodes, 'processes', 'order', Ordering of data_models 'nfreqwin', Number of frequency windows in simulation 'ntimes', Number of hour angles in simulation 'rmax', Maximum radius of stations used in simulation (m) 'facets', Number of facets in deconvolution and imaging 'wprojection_planes', Number of wprojection planes 'vis_slices', Number of visibility slices (per Visibbility) 'npixel', Number of pixels in image 'cellsize', Cellsize in radians 'seed', Random number seed 'dirty_max', Maximum in dirty image 'dirty_min', Minimum in dirty image 'psf_max', 'psf_min', 'restored_max', 'restored_min', 'deconvolved_max', 'deconvolved_min', 'residual_max', 'residual_min', 'git_info', GIT hash (not definitive since local mods are possible) :param results: Initial state :param seed: Random number seed (used in gain simulations) :param context: imaging context :param context: Type of context: '2d'|'timeslice'|'wstack' :param nworkers: Number of dask workers to use :param threads_per_worker: Number of threads per worker :param processes: Use processes instead of threads 'processes'|'threads' :param order: See simulate_list_list_arlexecute_workflow_workflowkflow :param nfreqwin: See simulate_list_list_arlexecute_workflow_workflowkflow :param ntimes: See simulate_list_list_arlexecute_workflow_workflowkflow :param rmax: See simulate_list_list_arlexecute_workflow_workflowkflow :param facets: Number of facets to use :param wprojection_planes: Number of wprojection planes to use :param use_dask: Use dask or immediate evaluation :param kwargs: :return: results dictionary """ numpy.random.seed(seed) results['seed'] = seed start_all = time.time() results['context'] = context results['hostname'] = socket.gethostname() results['git_hash'] = git_hash() results['epoch'] = time.strftime("%Y-%m-%d %H:%M:%S") zerow = False print("Context is %s" % context) results['nworkers'] = nworkers results['threads_per_worker'] = threads_per_worker results['processes'] = processes results['memory'] = memory results['order'] = order results['nfreqwin'] = nfreqwin results['ntimes'] = ntimes results['rmax'] = rmax results['facets'] = facets results['wprojection_planes'] = wprojection_planes results['use_dask'] = use_dask print("At start, configuration is {0!r}".format(results)) # Parameters determining scale frequency = numpy.linspace(0.8e8, 1.2e8, nfreqwin) centre = nfreqwin // 2 if nfreqwin > 1: channel_bandwidth = numpy.array(nfreqwin * [frequency[1] - frequency[0]]) else: channel_bandwidth = numpy.array([1e6]) times = numpy.linspace(-numpy.pi / 3.0, numpy.pi / 3.0, ntimes) phasecentre = SkyCoord(ra=+30.0 * u.deg, dec=-60.0 * u.deg, frame='icrs', equinox='J2000') if use_dask: client = get_dask_Client(threads_per_worker=threads_per_worker, memory_limit=memory * 1024 * 1024 * 1024, n_workers=nworkers) arlexecute.set_client(client) nodes = findNodes(arlexecute.client) unodes = list(numpy.unique(nodes)) results['nnodes'] = len(unodes) print("Defined %d workers on %d nodes" % (nworkers, results['nnodes'])) print("Workers are: %s" % str(nodes)) else: arlexecute.set_client(use_dask=use_dask) results['nnodes'] = 1 unodes = None vis_list = simulate_list_arlexecute_workflow( 'LOWBD2', frequency=frequency, channel_bandwidth=channel_bandwidth, times=times, phasecentre=phasecentre, order=order, format='blockvis', rmax=rmax) print("****** Visibility creation ******") vis_list = arlexecute.persist(vis_list) # Find the best imaging parameters but don't bring the vis_list back here def get_wf(bv): v = convert_blockvisibility_to_visibility(bv) return advise_wide_field(v, guard_band_image=6.0, delA=0.02, facets=facets, wprojection_planes=wprojection_planes, oversampling_synthesised_beam=4.0) wprojection_planes = 1 advice = arlexecute.compute(arlexecute.execute(get_wf)(vis_list[0]), sync=True) npixel = advice['npixels2'] cellsize = advice['cellsize'] if context == 'timeslice': vis_slices = ntimes print("Using timeslice with %d slices" % vis_slices) elif context == '2d': vis_slices = 1 else: context = 'wstack' vis_slices = 5 * advice['vis_slices'] print("Using wstack with %d slices" % vis_slices) results['vis_slices'] = vis_slices results['cellsize'] = cellsize results['npixel'] = npixel gleam_model_list = [ arlexecute.execute(create_low_test_image_from_gleam)( npixel=npixel, frequency=[frequency[f]], channel_bandwidth=[channel_bandwidth[f]], cellsize=cellsize, phasecentre=phasecentre, polarisation_frame=PolarisationFrame("stokesI"), flux_limit=0.3, applybeam=True) for f, freq in enumerate(frequency) ] start = time.time() print("****** Starting GLEAM model creation ******") gleam_model_list = arlexecute.compute(gleam_model_list, sync=True) cmodel = smooth_image(gleam_model_list[centre]) export_image_to_fits(cmodel, "pipelines-timings-arlexecute-gleam_cmodel.fits") end = time.time() results['time create gleam'] = end - start print("Creating GLEAM model took %.2f seconds" % (end - start)) gleam_model_list = arlexecute.scatter(gleam_model_list) vis_list = predict_list_arlexecute_workflow(vis_list, gleam_model_list, vis_slices=vis_slices, context=context) start = time.time() print("****** Starting GLEAM model visibility prediction ******") vis_list = arlexecute.compute(vis_list, sync=True) end = time.time() results['time predict'] = end - start print("GLEAM model Visibility prediction took %.2f seconds" % (end - start)) # Corrupt the visibility for the GLEAM model print("****** Visibility corruption ******") vis_list = corrupt_list_arlexecute_workflow(vis_list, phase_error=1.0) start = time.time() vis_list = arlexecute.compute(vis_list, sync=True) vis_list = arlexecute.scatter(vis_list) end = time.time() results['time corrupt'] = end - start print("Visibility corruption took %.2f seconds" % (end - start)) # Create an empty model image model_list = [ arlexecute.execute(create_image_from_visibility)( vis_list[f], npixel=npixel, cellsize=cellsize, frequency=[frequency[f]], channel_bandwidth=[channel_bandwidth[f]], polarisation_frame=PolarisationFrame("stokesI")) for f, freq in enumerate(frequency) ] model_list = arlexecute.compute(model_list, sync=True) model_list = arlexecute.scatter(model_list) psf_list = invert_list_arlexecute_workflow(vis_list, model_list, vis_slices=vis_slices, context=context, facets=facets, dopsf=True) start = time.time() print("****** Starting PSF calculation ******") psf, sumwt = arlexecute.compute(psf_list, sync=True)[centre] end = time.time() results['time psf invert'] = end - start print("PSF invert took %.2f seconds" % (end - start)) results['psf_max'] = qa_image(psf).data['max'] results['psf_min'] = qa_image(psf).data['min'] dirty_list = invert_list_arlexecute_workflow(vis_list, model_list, vis_slices=vis_slices, context=context, facets=facets) start = time.time() print("****** Starting dirty image calculation ******") dirty, sumwt = arlexecute.compute(dirty_list, sync=True)[centre] end = time.time() results['time invert'] = end - start print("Dirty image invert took %.2f seconds" % (end - start)) print("Maximum in dirty image is ", numpy.max(numpy.abs(dirty.data)), ", sumwt is ", sumwt) qa = qa_image(dirty) results['dirty_max'] = qa.data['max'] results['dirty_min'] = qa.data['min'] # Create the ICAL pipeline to run 5 major cycles, starting selfcal at cycle 1. A global solution across all # frequencies (i.e. Visibilities) is performed. start = time.time() print("****** Starting ICAL ******") controls = create_calibration_controls() controls['T']['first_selfcal'] = 1 controls['G']['first_selfcal'] = 3 controls['B']['first_selfcal'] = 4 controls['T']['timescale'] = 'auto' controls['G']['timescale'] = 'auto' controls['B']['timescale'] = 1e5 if nfreqwin > 6: nmoment = 3 algorithm = 'mmclean' elif nfreqwin > 2: nmoment = 2 algorithm = 'mmclean' else: nmoment = 1 algorithm = 'msclean' start = time.time() ical_list = ical_list_arlexecute_workflow(vis_list, model_imagelist=model_list, context='wstack', calibration_context='TG', controls=controls, scales=[0, 3, 10], algorithm=algorithm, nmoment=nmoment, niter=1000, fractional_threshold=0.1, threshold=0.1, nmajor=5, gain=0.25, vis_slices=vis_slices, timeslice='auto', global_solution=False, psf_support=64, do_selfcal=True) end = time.time() results['time ICAL graph'] = end - start print("Construction of ICAL graph took %.2f seconds" % (end - start)) # Execute the graph start = time.time() result = arlexecute.compute(ical_list, sync=True) deconvolved, residual, restored = result end = time.time() results['time ICAL'] = end - start print("ICAL graph execution took %.2f seconds" % (end - start)) qa = qa_image(deconvolved[centre]) results['deconvolved_max'] = qa.data['max'] results['deconvolved_min'] = qa.data['min'] export_image_to_fits(deconvolved[centre], "pipelines-timings-arlexecute-ical_deconvolved.fits") qa = qa_image(residual[centre][0]) results['residual_max'] = qa.data['max'] results['residual_min'] = qa.data['min'] export_image_to_fits(residual[centre][0], "pipelines-timings-arlexecute-ical_residual.fits") qa = qa_image(restored[centre]) results['restored_max'] = qa.data['max'] results['restored_min'] = qa.data['min'] export_image_to_fits(restored[centre], "pipelines-timings-arlexecute-ical_restored.fits") # arlexecute.close() end_all = time.time() results['time overall'] = end_all - start_all print("At end, results are {0!r}".format(results)) return results
def main(): """Run the workflow.""" init_logging() LOG.info("Starting imaging-pipeline") # Read parameters PARFILE = 'parameters.json' if len(sys.argv) > 1: PARFILE = sys.argv[1] LOG.info("JSON parameter file = %s", PARFILE) try: with open(PARFILE, "r") as par_file: jspar = json.load(par_file) except AssertionError as error: LOG.critical('ERROR %s', error) return # We will use dask arlexecute.set_client(get_dask_Client()) arlexecute.run(init_logging) # Import visibility list from HDF5 file vis_list = import_blockvisibility_from_hdf5( '%s/%s' % (RESULTS_DIR, jspar["files"]["vis_list"])) # Now read the BlockVisibilities constructed using a model drawn from GLEAM predicted_vislist = import_blockvisibility_from_hdf5( '%s/%s' % (RESULTS_DIR, jspar["files"]["predicted_vis_list"])) corrupted_vislist = import_blockvisibility_from_hdf5( '%s/%s' % (RESULTS_DIR, jspar["files"]["corrupted_vis_list"])) # Reproduce parameters from the visibility data ntimes = vis_list[0].nvis phasecentre = vis_list[0].phasecentre print(phasecentre) polframe = vis_list[0].polarisation_frame.type LOG.info("Polarisation Frame of vis_list: %s", polframe) wprojection_planes = jspar["advice"]["wprojection_planes"] guard_band_image = jspar["advice"]["guard_band_image"] delA = jspar["advice"]["delA"] advice_low = advise_wide_field(vis_list[0], guard_band_image=guard_band_image, delA=delA, wprojection_planes=wprojection_planes) advice_high = advise_wide_field(vis_list[-1], guard_band_image=guard_band_image, delA=delA, wprojection_planes=wprojection_planes) vis_slices = advice_low['vis_slices'] npixel = advice_high['npixels2'] cellsize = min(advice_low['cellsize'], advice_high['cellsize']) # Recovering frequencies fstart = vis_list[0].frequency fend = vis_list[-1].frequency num_freq_win = len(vis_list) frequency = numpy.linspace(fstart, fend, num_freq_win) # Recovering bandwidths channel_bandwidth = numpy.array( num_freq_win * [vis_list[1].frequency - vis_list[0].frequency]) # Get the LSM. This is currently blank. model_list = [ arlexecute.execute(create_image_from_visibility)( vis_list[f], npixel=npixel, frequency=[frequency[f]], channel_bandwidth=[channel_bandwidth[f]], cellsize=cellsize, phasecentre=phasecentre, polarisation_frame=PolarisationFrame(polframe)) for f, freq in enumerate(frequency) ] # future_predicted_vislist = arlexecute.scatter(predicted_vislist) # Create and execute graphs to make the dirty image and PSF # LOG.info('About to run invert to get dirty image') # dirty_list = invert_component(future_predicted_vislist, model_list, # context='wstack', # vis_slices=vis_slices, dopsf=False) # dirty_list = arlexecute.compute(dirty_list, sync=True) # LOG.info('About to run invert to get PSF') # psf_list = invert_component(future_predicted_vislist, model_list, # context='wstack', # vis_slices=vis_slices, dopsf=True) # psf_list = arlexecute.compute(psf_list, sync=True) # Now deconvolve using msclean # LOG.info('About to run deconvolve') # deconvolve_list, _ = deconvolve_component( # dirty_list, psf_list, # model_imagelist=model_list, # deconvolve_facets=8, # deconvolve_overlap=16, # deconvolve_taper='tukey', # scales=[0, 3, 10], # algorithm='msclean', # niter=1000, # fractional_threshold=0.1, # threshold=0.1, # gain=0.1, # psf_support=64) # deconvolved = arlexecute.compute(deconvolve_list, sync=True) LOG.info('About to run continuum imaging') continuum_imaging_list = continuum_imaging_arlexecute( predicted_vislist, model_imagelist=model_list, context=jspar["processing"]["continuum_imaging"] ["context"], #'wstack', vis_slices=vis_slices, scales=jspar["processing"]["continuum_imaging"] ["scales"], #[0, 3, 10], algorithm=jspar["processing"]["continuum_imaging"] ["algorithm"], #'mmclean', nmoment=jspar["processing"]["continuum_imaging"]["nmoment"], #3, niter=jspar["processing"]["continuum_imaging"]["niter"], #1000, fractional_threshold=jspar["processing"]["continuum_imaging"] ["fractional_threshold"], #0.1, threshold=jspar["processing"]["continuum_imaging"]["threshold"], #0.1, nmajor=jspar["processing"]["continuum_imaging"]["nmajor"], #5, gain=jspar["processing"]["continuum_imaging"]["gain"], #0.25, deconvolve_facets=jspar["processing"]["continuum_imaging"] ["deconvolve_facets"], #8, deconvolve_overlap=jspar["processing"]["continuum_imaging"] ["deconvolve_overlap"], #16, deconvolve_taper=jspar["processing"]["continuum_imaging"] ["deconvolve_taper"], #'tukey', psf_support=jspar["processing"]["continuum_imaging"] ["psf_support"]) #64) result = arlexecute.compute(continuum_imaging_list, sync=True) deconvolved = result[0][0] residual = result[1][0] restored = result[2][0] print(qa_image(deconvolved, context='Clean image - no selfcal')) print(qa_image(restored, context='Restored clean image - no selfcal')) export_image_to_fits( restored, '%s/%s' % (RESULTS_DIR, jspar["files"]["continuum_imaging_restored"])) print(qa_image(residual[0], context='Residual clean image - no selfcal')) export_image_to_fits( residual[0], '%s/%s' % (RESULTS_DIR, jspar["files"]["continuum_imaging_residual"])) controls = create_calibration_controls() controls['T']['first_selfcal'] = jspar["processing"]["controls"]["T"][ "first_selfcal"] controls['G']['first_selfcal'] = jspar["processing"]["controls"]["G"][ "first_selfcal"] controls['B']['first_selfcal'] = jspar["processing"]["controls"]["B"][ "first_selfcal"] controls['T']['timescale'] = jspar["processing"]["controls"]["T"][ "timescale"] controls['G']['timescale'] = jspar["processing"]["controls"]["G"][ "timescale"] controls['B']['timescale'] = jspar["processing"]["controls"]["B"][ "timescale"] PP.pprint(controls) future_corrupted_vislist = arlexecute.scatter(corrupted_vislist) ical_list = ical_arlexecute( future_corrupted_vislist, model_imagelist=model_list, context=jspar["processing"]["ical"]["context"], #'wstack', calibration_context=jspar["processing"]["ical"] ["calibration_context"], #'TG', controls=controls, vis_slices=ntimes, scales=jspar["processing"]["ical"]["scales"], #[0, 3, 10], timeslice=jspar["processing"]["ical"]["timeslice"], #'auto', algorithm=jspar["processing"]["ical"]["algorithm"], #'mmclean', nmoment=jspar["processing"]["ical"]["nmoment"], #3, niter=jspar["processing"]["ical"]["niter"], #1000, fractional_threshold=jspar["processing"]["ical"] ["fractional_threshold"], #0.1, threshold=jspar["processing"]["ical"]["threshold"], #0.1, nmajor=jspar["processing"]["ical"]["nmajor"], #5, gain=jspar["processing"]["ical"]["gain"], #0.25, deconvolve_facets=jspar["processing"]["ical"] ["deconvolve_facets"], #8, deconvolve_overlap=jspar["processing"]["ical"] ["deconvolve_overlap"], #16, deconvolve_taper=jspar["processing"]["ical"] ["deconvolve_taper"], #'tukey', global_solution=jspar["processing"]["ical"] ["global_solution"], #False, do_selfcal=jspar["processing"]["ical"]["do_selfcal"], #True, psf_support=jspar["processing"]["ical"]["psf_support"]) #64 LOG.info('About to run ical') result = arlexecute.compute(ical_list, sync=True) deconvolved = result[0][0] residual = result[1][0] restored = result[2][0] print(qa_image(deconvolved, context='Clean image')) print(qa_image(restored, context='Restored clean image')) export_image_to_fits( restored, '%s/%s' % (RESULTS_DIR, jspar["files"]["ical_restored"])) print(qa_image(residual[0], context='Residual clean image')) export_image_to_fits( residual[0], '%s/%s' % (RESULTS_DIR, jspar["files"]["ical_residual"])) arlexecute.close()
def ical_serial(block_vis: BlockVisibility, model: Image, components=None, context='2d', controls=None, **kwargs): """ Post observation image, deconvolve, and self-calibrate :param vis: :param model: Model image :param components: Initial components :param context: Imaging context :param controls: calibration controls dictionary :return: model, residual, restored """ nmajor = get_parameter(kwargs, 'nmajor', 5) log.info("ical_serial: Performing %d major cycles" % nmajor) do_selfcal = get_parameter(kwargs, "do_selfcal", False) if controls is None: controls = create_calibration_controls(**kwargs) # The model is added to each major cycle and then the visibilities are # calculated from the full model vis = convert_blockvisibility_to_visibility(block_vis) block_vispred = copy_visibility(block_vis, zero=True) vispred = convert_blockvisibility_to_visibility(block_vispred) vispred.data['vis'][...] = 0.0 visres = copy_visibility(vispred) vispred = predict_serial(vispred, model, context=context, **kwargs) if components is not None: vispred = predict_skycomponent_visibility(vispred, components) if do_selfcal: vis, gaintables = calibrate_function(vis, vispred, 'TGB', controls, iteration=-1) visres.data['vis'] = vis.data['vis'] - vispred.data['vis'] dirty, sumwt = invert_serial(visres, model, context=context, **kwargs) log.info("Maximum in residual image is %.6f" % (numpy.max(numpy.abs(dirty.data)))) psf, sumwt = invert_serial(visres, model, dopsf=True, context=context, **kwargs) thresh = get_parameter(kwargs, "threshold", 0.0) for i in range(nmajor): log.info("ical_serial: Start of major cycle %d of %d" % (i, nmajor)) cc, res = deconvolve_cube(dirty, psf, **kwargs) model.data += cc.data vispred.data['vis'][...] = 0.0 vispred = predict_serial(vispred, model, context=context, **kwargs) if do_selfcal: vis, gaintables = calibrate_function(vis, vispred, 'TGB', controls, iteration=i) visres.data['vis'] = vis.data['vis'] - vispred.data['vis'] dirty, sumwt = invert_serial(visres, model, context=context, **kwargs) log.info("Maximum in residual image is %s" % (numpy.max(numpy.abs(dirty.data)))) if numpy.abs(dirty.data).max() < 1.1 * thresh: log.info("ical_serial: Reached stopping threshold %.6f Jy" % thresh) break log.info("ical_serial: End of major cycle") log.info("ical_serial: End of major cycles") restored = restore_cube(model, psf, dirty, **kwargs) return model, dirty, restored