def test_ical_pipeline_global(self):
self.actualSetUp(add_errors=True)
controls = create_calibration_controls()
controls['T']['first_selfcal'] = 1
controls['T']['timescale'] = 'auto'
clean, residual, restored, gt_list = \
ical_list_serial_workflow(self.vis_list,
model_imagelist=self.model_imagelist,
context='2d',
algorithm='mmclean', facets=1,
scales=[0, 3, 10],
niter=1000, fractional_threshold=0.1, threshold=0.1,
nmoment=3,
nmajor=5, gain=0.1,
deconvolve_facets=4, deconvolve_overlap=32,
deconvolve_taper='tukey', psf_support=64,
calibration_context='T', controls=controls, do_selfcal=True,
global_solution=True)
centre = len(clean) // 2
if self.persist:
export_image_to_fits(clean[centre], '%s/test_pipelines_ical_global_pipeline_serial_clean.fits' % self.dir)
export_image_to_fits(residual[centre][0], '%s/test_pipelines_ical_global_pipeline_serial_residual.fits' % self.dir)
export_image_to_fits(restored[centre], '%s/test_pipelines_ical_global_pipeline_serial_restored.fits' % self.dir)
export_gaintable_to_hdf5(gt_list[0]['T'],
'%s/test_pipelines_ical_global_pipeline_serial_gaintable.hdf5' %
self.dir)
qa = qa_image(restored[centre])
assert numpy.abs(qa.data['max'] - 98.92656340122159) < 1.0, str(qa)
assert numpy.abs(qa.data['min'] + 0.7024492707920869) < 1.0, str(qa)
def test_continuum_imaging_pipeline(self):
self.actualSetUp(add_errors=False, zerow=True)
clean, residual, restored = \
continuum_imaging_list_serial_workflow(self.vis_list,
model_imagelist=self.model_imagelist,
context='2d',
algorithm='mmclean', facets=1,
scales=[0, 3, 10],
niter=1000, fractional_threshold=0.1, threshold=0.1,
nmoment=3,
nmajor=5, gain=0.1,
deconvolve_facets=4, deconvolve_overlap=32,
deconvolve_taper='tukey', psf_support=64)
centre = len(clean) // 2
if self.persist:
export_image_to_fits(
clean[centre],
'%s/test_pipelines_continuum_imaging_pipeline_serial_clean.fits'
% self.dir)
export_image_to_fits(
residual[centre][0],
'%s/test_pipelines_continuum_imaging_pipeline_serial_residual.fits'
% self.dir)
export_image_to_fits(
restored[centre],
'%s/test_pipelines_continuum_imaging_pipeline_serial_restored.fits'
% self.dir)
qa = qa_image(restored[centre])
assert numpy.abs(qa.data['max'] - 100.13762476849081) < 1.0, str(qa)
assert numpy.abs(qa.data['min'] + 0.03627273884170454) < 1.0, str(qa)
def test_residual_list(self):
self.actualSetUp(zerow=True)
centre = self.freqwin // 2
residual_image_list = residual_list_serial_workflow(self.vis_list,
self.model_list,
context='2d')
qa = qa_image(residual_image_list[centre][0])
assert numpy.abs(qa.data['max'] - 0.35139716991480785) < 1.0, str(qa)
assert numpy.abs(qa.data['min'] + 0.7681701460717593) < 1.0, str(qa)
def progress(res, tl_list, gt_list, it, context='MPCCAL'):
print('Iteration %d' % it)
print(
qa_image(res,
context='%s residual image: iteration %d' %
(context, it)))
export_image_to_fits(
res,
arl_path(
"test_results/low-sims-mpc-%s-residual_iteration%d_rmax%.1f.fits"
% (context, it, rmax)))
show_image(res,
title='%s residual image: iteration %d' % (context, it))
plt.show(block=block_plots)
combined_model = calculate_skymodel_equivalent_image(tl_list)
print(
qa_image(combined_model,
context='Combined model: iteration %d' % it))
export_image_to_fits(
combined_model,
arl_path(
"test_results/low-sims-mpc-%s-model_iteration%d_rmax%.1f.fits"
% (context, it, rmax)))
plt.clf()
for i in range(len(tl_list)):
plt.plot(
numpy.angle(tl_list[i].gaintable.gain[:, :, 0, 0,
0]).flatten(),
numpy.angle(gt_list[i]['T'].gain[:, :, 0, 0, 0]).flatten(),
'.')
plt.xlabel('Current phase')
plt.ylabel('Update to phase')
plt.title("%s iteration%d: Change in phase" % (context, it))
plt.savefig(
arl_path(
"test_results/low-sims-mpc-%s-skymodel-phase-change_iteration%d.jpg"
% (context, it)))
plt.show(block=block_plots)
return tl_list
def test_random(self):
error_dirty, sumwt = self.simulation(self.get_args(), '')
qa = qa_image(error_dirty)
numpy.testing.assert_almost_equal(qa.data['max'],
2.2055849698035616e-06, 12)
numpy.testing.assert_almost_equal(qa.data['min'],
-6.838117387793031e-07, 12)
numpy.testing.assert_almost_equal(qa.data['rms'],
3.7224203394509413e-07, 12)
def test_wind(self):
error_dirty, sumwt = self.simulation(self.get_args(), 'wind')
qa = qa_image(error_dirty)
numpy.testing.assert_almost_equal(qa.data['max'],
1.2440965953105578e-06, 12)
numpy.testing.assert_almost_equal(qa.data['min'],
-3.836051633637655e-06, 12)
numpy.testing.assert_almost_equal(qa.data['rms'],
5.840397284050296e-07, 12)
def test_gravity(self):
if os.path.isdir(arl_path('models/interpolated')):
error_dirty, sumwt = self.simulation(self.get_args(), 'gravity')
qa = qa_image(error_dirty)
numpy.testing.assert_almost_equal(qa.data['max'],
2.2055849698035616e-06, 12)
numpy.testing.assert_almost_equal(qa.data['min'],
-6.838117387793031e-07, 12)
numpy.testing.assert_almost_equal(qa.data['rms'],
3.7224203394509413e-07, 12)
def test_ical_pipeline(self):
amp_errors = {'T': 0.0, 'G': 0.00, 'B': 0.0}
phase_errors = {'T': 0.1, 'G': 0.0, 'B': 0.0}
self.actualSetUp(add_errors=True,
block=True,
amp_errors=amp_errors,
phase_errors=phase_errors)
controls = create_calibration_controls()
controls['T']['first_selfcal'] = 1
controls['G']['first_selfcal'] = 3
controls['B']['first_selfcal'] = 4
controls['T']['timescale'] = 'auto'
controls['G']['timescale'] = 'auto'
controls['B']['timescale'] = 1e5
clean, residual, restored = \
ical_list_serial_workflow(self.vis_list, model_imagelist=self.model_imagelist, context='2d',
calibration_context='T', controls=controls, do_selfcal=True,
global_solution=False,
algorithm='mmclean',
facets=1,
scales=[0, 3, 10],
niter=1000, fractional_threshold=0.1,
nmoments=2, nchan=self.freqwin,
threshold=2.0, nmajor=5, gain=0.1,
deconvolve_facets=8, deconvolve_overlap=16, deconvolve_taper='tukey')
centre = len(clean) // 2
export_image_to_fits(
clean[centre],
'%s/test_pipelines_ical_pipeline_clean.fits' % self.dir)
export_image_to_fits(
residual[centre][0],
'%s/test_pipelines_ical_pipeline_residual.fits' % self.dir)
export_image_to_fits(
restored[centre],
'%s/test_pipelines_ical_pipeline_restored.fits' % self.dir)
qa = qa_image(restored[centre])
assert numpy.abs(qa.data['max'] - 100.13739440876233) < 1.0, str(qa)
assert numpy.abs(qa.data['min'] + 0.03644435471804354) < 1.0, str(qa)
elif context == 'wstack':
future = invert_list_arlexecute_workflow(vt_list, targetimage_list, context=context, vis_slices=31)
else:
future = invert_list_arlexecute_workflow(vt_list, targetimage_list, context=context)
result = arlexecute.compute(future, sync=True)
targetimage = result[0][0]
show_image(targetimage)
plt.title(context)
plt.savefig('2_'+context+'.jpg')
#plt.show()
print("Dirty Image %s" % qa_image(targetimage, context="imaging-fits notebook, using processor %s" % context))
export_image_to_fits(targetimage, '%s/imaging-fits_dirty_%s.fits' % (results_dir, context))
comps = find_skycomponents(targetimage, fwhm=1.0, threshold=10.0, npixels=5)
plt.clf()
for comp in comps:
distance = comp.direction.separation(model.phasecentre)
dft_flux = sum_visibility(vt, comp.direction)[0]
err = (comp.flux[0, 0] - dft_flux) / dft_flux
plt.plot(distance, err, '.', color='r')
plt.ylabel('Fractional error of image vs DFT')
plt.xlabel('Distance from phasecentre (deg)')
plt.title(
"Fractional error in %s recovered flux vs distance from phasecentre" %
context)
# Create and execute graphs to make the dirty image and PSF
# In[ ]:
end = time.time()
log.info('%d: invert finished' % (rank))
print('%d: invert finished in %f seconds' % (rank, end - start), flush=True)
if rank == 0:
#print("sumwts",flush=True)
#print(dirty_list[0][1])
log.info('After invert to get dirty image')
dirty = dirty_list[0][0]
#show_image(dirty, cm='Greys', vmax=1.0, vmin=-0.1)
#plt.show()
print(qa_image(dirty))
export_image_to_fits(dirty, '%s/imaging-dirty.fits' % (results_dir))
log.info('After invert to get PSF')
psf = psf_list[0][0]
#show_image(psf, cm='Greys', vmax=0.1, vmin=-0.01)
#plt.show()
print(qa_image(psf))
export_image_to_fits(psf, '%s/imaging-psf.fits' % (results_dir))
# Now deconvolve using msclean
# In[ ]:
log.info('%d: About to run deconvolve' % (rank))
print('%d: About to run deconvolve' % (rank), flush=True)
else:
d, sumwt = invert_list_serial_workflow([v], [m],
context=context,
dopsf=False,
vis_slices=vis_slices)[0]
p, sumwt = invert_list_serial_workflow([v], [m],
context=context,
dopsf=True,
vis_slices=vis_slices)[0]
c, resid = deconvolve_cube(d,
p,
m,
threshold=0.01,
fracthresh=0.01,
window_shape='quarter',
niter=100,
gain=0.1,
algorithm='hogbom-complex')
r = restore_cube(c, p, resid, psfwidth=psfwidth)
return r
print('About assemble cubes and deconvolve each frequency')
restored_list = [load_invert_and_deconvolve(c) for c in range(nchan)]
restored_cube = image_gather_channels(restored_list)
print("Processing took %.3f s" % (time.time() - start))
print(qa_image(restored_cube, context='CLEAN restored cube'))
export_image_to_fits(
restored_cube, '%s/dprepb_serial_%s_clean_restored_cube.fits' %
(results_dir, context))
progress,
context='ICAL'),
nmajor=args.ical_nmajor,
context='2d',
algorithm='hogbom',
fractional_threshold=0.3,
threshold=0.1,
gain=0.1,
niter=1000,
psf_support=512,
deconvolve_facets=8,
deconvolve_overlap=16,
deconvolve_taper='tukey')
(ical_skymodel, residual) = arlexecute.compute(result, sync=True)
print(qa_image(residual, context='ICAL residual image'))
print('ical finished')
combined_model = calculate_skymodel_equivalent_image(ical_skymodel)
print(qa_image(combined_model, context='ICAL combined model'))
psf_obs = invert_list_arlexecute_workflow([future_vis], [future_model],
context='2d',
dopsf=True)
result = restore_list_arlexecute_workflow([combined_model], psf_obs,
[(residual, 0.0)])
result = arlexecute.compute(result, sync=True)
ical_restored = result[0]
export_image_to_fits(
ical_restored,
chunks = [all_skymodel[i:i + ngroup] for i in range(0, len(all_skymodel), ngroup)]
for chunk in chunks:
result = predict_skymodel_list_arlexecute_workflow(future_vis, chunk, context='2d', docal=True)
work_vis = arlexecute.compute(result, sync=True)
for w in work_vis:
all_skymodel_vis.data['vis'] += w.data['vis']
assert numpy.max(numpy.abs(all_skymodel_vis.data['vis'])) > 0.0
all_skymodel_blockvis = convert_visibility_to_blockvisibility(all_skymodel_vis)
#######################################################################################################
# Now proceed to run MPCCAL in ICAL mode i.e. with only one skymodel
def progress(res, tl_list, gt_list, it, context='MPCCAL'):
print('Iteration %d' % it)
print(qa_image(res, context='%s residual image: iteration %d' % (context, it)))
export_image_to_fits(res, arl_path("test_results/low-sims-mpc-%s-residual_iteration%d_rmax%.1f.fits" %
(context, it, rmax)))
show_image(res, title='%s residual image: iteration %d' % (context, it))
plt.show(block=block_plots)
combined_model = calculate_skymodel_equivalent_image(tl_list)
print(qa_image(combined_model, context='Combined model: iteration %d' % it))
export_image_to_fits(combined_model, arl_path("test_results/low-sims-mpc-%s-model_iteration%d_rmax%.1f.fits" %
(context, it, rmax)))
plt.clf()
for i in range(len(tl_list)):
plt.plot(numpy.angle(tl_list[i].gaintable.gain[:, :, 0, 0, 0]).flatten(),
numpy.angle(gt_list[i]['T'].gain[:, :, 0, 0, 0]).flatten(),
'.')
context='wstack',
vis_slices=vis_slices,
dopsf=False)
psf_list = invert_list_mpi_workflow(predicted_vislist,
model_list,
context='wstack',
vis_slices=vis_slices,
dopsf=True)
# Create and execute graphs to make the dirty image and PSF
# In[ ]:
if rank == 0:
print("sumwts", flush=True)
print(dirty_list[0][1])
log.info('About to run invert to get dirty image')
dirty = dirty_list[0][0]
#show_image(dirty, cm='Greys', vmax=1.0, vmin=-0.1)
#plt.show()
print(qa_image(dirty))
export_image_to_fits(dirty, '%s/imaging-dirty.fits' % (results_dir))
log.info('About to run invert to get PSF')
psf = psf_list[0][0]
#show_image(psf, cm='Greys', vmax=0.1, vmin=-0.01)
#plt.show()
print(qa_image(psf))
export_image_to_fits(psf, '%s/imaging-psf.fits' % (results_dir))
vt = convert_blockvisibility_to_visibility(bvt)
vt = convert_visibility_to_stokes(vt)
cellsize = 20.0 * numpy.pi / (180.0 * 3600.0)
npixel = 512
model = create_image_from_visibility(vt, cellsize=cellsize, npixel=npixel,
polarisation_frame=PolarisationFrame('stokesIQUV'))
dirty, sumwt = invert_list_serial_workflow([vt], [model], context='2d')[0]
psf, sumwt = invert_list_serial_workflow([vt], [model], context='2d', dopsf=True)[0]
export_image_to_fits(dirty, '%s/compare_imaging_sim2_dirty.fits' % (results_dir))
export_image_to_fits(psf, '%s/compare_imaging_sim2_psf.fits' % (results_dir))
# Deconvolve using clean
comp, residual = deconvolve_cube(dirty, psf, niter=10000, threshold=0.001, fractional_threshold=0.001,
window_shape='quarter', gain=0.7, scales=[0, 3, 10, 30])
restored = restore_cube(comp, psf, residual)
export_image_to_fits(restored, '%s/compare_imaging_sim2_restored.fits' % (results_dir))
export_image_to_fits(residual, '%s/compare_imaging_sim2_residual.fits' % (results_dir))
qa = qa_image(restored)
assert numpy.abs(qa.data['max'] - 1.006140596404203) < 1e-7, qa
assert numpy.abs(qa.data['maxabs'] - 1.006140596404203) < 1e-7, qa
assert numpy.abs(qa.data['min'] + 0.23890808520954754) < 1e-7, qa
assert numpy.abs(qa.data['rms'] - 0.007366519782047875) < 1e-7, qa
assert numpy.abs(qa.data['medianabs'] - 0.0005590537883509844) < 1e-7, qa
except ModuleNotFoundError as error:
print(error.__class__.__name__ + ": " + error.msg)
pass
