def _predict_base(self,
context='2d',
extra='',
fluxthreshold=1.0,
facets=1,
vis_slices=1,
gcfcf=None,
**kwargs):
centre = self.freqwin // 2
vis_list = zero_list_serial_workflow(self.vis_list)
vis_list = predict_list_serial_workflow(vis_list,
self.model_list,
context=context,
vis_slices=vis_slices,
facets=facets,
gcfcf=gcfcf,
**kwargs)
vis_list = subtract_list_serial_workflow(self.vis_list, vis_list)
dirty = invert_list_serial_workflow(vis_list,
self.model_list,
context='2d',
dopsf=False,
gcfcf=gcfcf,
normalize=True)[centre]
assert numpy.max(numpy.abs(dirty[0].data)), "Residual image is empty"
export_image_to_fits(
dirty[0], '%s/test_imaging_predict_%s%s_serial_dirty.fits' %
(self.dir, context, extra))
maxabs = numpy.max(numpy.abs(dirty[0].data))
assert maxabs < fluxthreshold, "Error %.3f greater than fluxthreshold %.3f " % (
maxabs, fluxthreshold)
def ft_cal_sm(ov, sm):
assert isinstance(ov, Visibility), ov
assert isinstance(sm, SkyModel), sm
v = copy_visibility(ov)
v.data['vis'][...] = 0.0 + 0.0j
if len(sm.components) > 0:
if isinstance(sm.mask, Image):
comps = copy_skycomponent(sm.components)
comps = apply_beam_to_skycomponent(comps, sm.mask)
v = predict_skycomponent_visibility(v, comps)
else:
v = predict_skycomponent_visibility(v, sm.components)
if isinstance(sm.image, Image):
if numpy.max(numpy.abs(sm.image.data)) > 0.0:
if isinstance(sm.mask, Image):
model = copy_image(sm.image)
model.data *= sm.mask.data
else:
model = sm.image
v = predict_list_serial_workflow([v], [model],
context=context,
vis_slices=vis_slices,
facets=facets,
gcfcf=gcfcf,
**kwargs)[0]
if docal and isinstance(sm.gaintable, GainTable):
bv = convert_visibility_to_blockvisibility(v)
bv = apply_gaintable(bv, sm.gaintable, inverse=True)
v = convert_blockvisibility_to_visibility(bv)
return v
def predict_list_mpi_workflow(vis_list, model_imagelist, context, vis_slices=1, facets=1,
gcfcf=None, comm=MPI.COMM_WORLD, **kwargs):
"""Predict, iterating over both the scattered vis_list and image
The visibility and image are scattered, the visibility is predicted on each part, and then the
parts are assembled. About data distribution: vis_list and model_imagelist
live in rank 0; vis_slices, facets, context are replicated in all nodes.
gcfcf if exists lives in rank 0; if not every mpi proc will create its own
for the corresponding subset of the image.
:param vis_list:
:param model_imagelist: Model used to determine image parameters
:param vis_slices: Number of vis slices (w stack or timeslice)
:param facets: Number of facets (per axis)
:param context: Type of processing e.g. 2d, wstack, timeslice or facets
:param gcfcg: tuple containing grid correction and convolution function
:param comm: MPI communicator
:param kwargs: Parameters for functions in components
:return: List of vis_lists
"""
rank = comm.Get_rank()
size = comm.Get_size()
log.info('%d: In predict_list_mpi_workflow: %d elements in vis_list' % (rank,len(vis_list)))
# the assert only makes sense in proc 0 as for the others both lists are
# empty
assert len(vis_list) == len(model_imagelist), "Model must be the same length as the vis_list"
# The use_serial_predict version paralelizes by freq (my opt version)
assert get_parameter(kwargs, "use_serial_predict", True),"Only freq paralellization implemented"
#if get_parameter(kwargs, "use_serial_predict", False):
if get_parameter(kwargs, "use_serial_predict", True):
from workflows.serial.imaging.imaging_serial import predict_list_serial_workflow
image_results_list = list()
# Distribute visibilities and model by freq
sub_vis_list= numpy.array_split(vis_list, size)
sub_vis_list=comm.scatter(sub_vis_list,root=0)
sub_model_imagelist= numpy.array_split(model_imagelist, size)
sub_model_imagelist=comm.scatter(sub_model_imagelist,root=0)
if gcfcf is not None:
sub_gcfcf = numpy.array_split(gcfcf,size)
sub_gcfcf=comm.scatter(sub_gcfcf,root=0)
isinstance(sub_vis_list[0], Visibility)
image_results_list= [predict_list_serial_workflow(vis_list=[sub_vis_list[i]],
model_imagelist=[sub_model_imagelist[i]], vis_slices=vis_slices,
facets=facets, context=context, gcfcf=gcfcf, **kwargs)[0]
for i, _ in enumerate(sub_vis_list)]
#print(image_results_list)
image_results_list=comm.gather(image_results_list,root=0)
if rank == 0:
#image_results_list_list=[x for x in image_results_list_list if x]
image_results_list=numpy.concatenate(image_results_list)
else:
image_results_list=list()
return image_results_list
def predict_skymodel_list_serial_workflow(vis_list,
skymodel_list,
context,
vis_slices=1,
facets=1,
gcfcf=None,
**kwargs):
"""Predict from a skymodel, iterating over both the vis_list and skymodel
The visibility and image are scattered, the visibility is predicted on each part, and then the
parts are assembled.
:param vis_list:
:param skymodel_list: skymodel list
:param vis_slices: Number of vis slices (w stack or timeslice)
:param facets: Number of facets (per axis)
:param context: Type of processing e.g. 2d, wstack, timeslice or facets
:param gcfcg: tuple containing grid correction and convolution function
:param kwargs: Parameters for functions in components
:return: List of vis_lists
"""
def extract_comps(sm):
return sm.components
def extract_image(sm):
return sm.images[0]
comp = [extract_comps(sm) for sm in skymodel_list]
images = [extract_image(sm) for sm in skymodel_list]
dft_vis_list = zero_list_serial_workflow(vis_list)
dft_vis_list = [
predict_skycomponent_visibility(dft_vis_list[i], comp[i])
for i, _ in enumerate(dft_vis_list)
]
fft_vis_list = zero_list_serial_workflow(vis_list)
fft_vis_list = predict_list_serial_workflow(fft_vis_list,
images,
context=context,
vis_slices=vis_slices,
facets=facets,
gcfcf=gcfcf,
**kwargs)
def vis_add(v1, v2):
vout = copy_visibility(v1)
vout.data['vis'] += v2.data['vis']
return vout
return [
vis_add(dft_vis_list[i], fft_vis_list[i])
for i, _ in enumerate(dft_vis_list)
]
def ft_ift_sm(ov, sm, g):
assert isinstance(ov, Visibility) or isinstance(ov,
BlockVisibility), ov
assert isinstance(sm, SkyModel), sm
if g is not None:
assert len(g) == 2, g
assert isinstance(g[0], Image), g[0]
assert isinstance(g[1], ConvolutionFunction), g[1]
v = copy_visibility(ov)
v.data['vis'][...] = 0.0 + 0.0j
if len(sm.components) > 0:
if isinstance(sm.mask, Image):
comps = copy_skycomponent(sm.components)
comps = apply_beam_to_skycomponent(comps, sm.mask)
v = predict_skycomponent_visibility(v, comps)
else:
v = predict_skycomponent_visibility(v, sm.components)
if isinstance(sm.image, Image):
if numpy.max(numpy.abs(sm.image.data)) > 0.0:
if isinstance(sm.mask, Image):
model = copy_image(sm.image)
model.data *= sm.mask.data
else:
model = sm.image
v = predict_list_serial_workflow([v], [model],
context=context,
vis_slices=vis_slices,
facets=facets,
gcfcf=[g],
**kwargs)[0]
assert isinstance(sm.image, Image), sm.image
result = invert_list_serial_workflow([v], [sm.image],
context=context,
vis_slices=vis_slices,
facets=facets,
gcfcf=[g],
**kwargs)[0]
if isinstance(sm.mask, Image):
result[0].data *= sm.mask.data
return result
def ft_ift_sm(ov, sm):
assert isinstance(ov, Visibility), ov
v = copy_visibility(ov)
v.data['vis'][...] = 0.0 + 0.0j
if len(sm.components) > 0:
if isinstance(sm.mask, Image):
comps = copy_skycomponent(sm.components)
comps = apply_beam_to_skycomponent(comps, sm.mask)
v = predict_skycomponent_visibility(v, comps)
else:
v = predict_skycomponent_visibility(v, sm.components)
if isinstance(sm.image, Image):
if numpy.max(numpy.abs(sm.image.data)) > 0.0:
if isinstance(sm.mask, Image):
model = copy_image(sm.image)
model.data *= sm.mask.data
else:
model = sm.image
v = predict_list_serial_workflow([v], [model],
context=context,
vis_slices=vis_slices,
facets=facets,
gcfcf=gcfcf,
**kwargs)[0]
assert isinstance(sm.image, Image), sm.image
result = invert_list_serial_workflow([v], [sm.image],
context=context,
vis_slices=vis_slices,
facets=facets,
gcfcf=gcfcf,
**kwargs)[0]
if isinstance(sm.mask, Image):
result[0].data *= sm.mask.data
return result
# each freqw during predict, it would safe time but use more space
gleam_model = comm.gather(sub_gleam_model, root=0)
if rank == 0:
gleam_model = numpy.concatenate(gleam_model)
else:
gleam_model = list()
# In[ ]:
original_predict = False
if original_predict:
if rank == 0:
log.info('About to run predict to get predicted visibility')
predicted_vislist = predict_list_serial_workflow(vis_list,
gleam_model,
context='wstack',
vis_slices=vis_slices)
else:
log.info('%d: About to run predict to get predicted visibility' % (rank))
print('%d: About to run predict to get predicted visibility' % (rank),
flush=True)
start = time.time()
# All procs call the function but only rank=0 gets the predicted_vislist
predicted_vislist = predict_list_mpi_workflow(vis_list,
gleam_model,
context='wstack',
vis_slices=vis_slices)
end = time.time()
#log.info('About to run corrupt to get corrupted visibility')
#corrupted_vislist = corrupt_list_serial_workflow(predicted_vislist, phase_error=1.0)
wprojection_planes=1
advice=advise_wide_field(block_vis, guard_band_image=4.0, delA=0.02,
wprojection_planes=wprojection_planes)
vis_slices = advice['vis_slices']
npixel=advice['npixels2']
cellsize=advice['cellsize']
gleam_model = create_low_test_image_from_gleam(npixel=npixel, frequency=frequency,
channel_bandwidth=channel_bandwidth, cellsize=cellsize, phasecentre=phasecentre, flux_limit = 1.0, applybeam=True)
predicted_vis = predict_list_serial_workflow(block_vis, gleam_model, vis_slices=51, context='wstack')
#print("np.sum(predicted_vis.data): ", numpy.sum(predicted_vis.data['vis']))
block_vis=convert_visibility_to_blockvisibility(predicted_vis)
#print("np.sum(block_vis.data): ", numpy.sum(block_vis.data['vis']))
#print("nchan npol nants ", block_vis.nchan, block_vis.npol, block_vis.nants)
#print("uvw", block_vis.uvw, numpy.sum(block_vis.uvw))
#print("vis", block_vis.vis, numpy.sum(block_vis.vis))
#print("weight", block_vis.weight, numpy.sum(block_vis.weight))
#print("time", block_vis.time, numpy.sum(block_vis.time))
#print("integration_time", block_vis.integration_time, numpy.sum(block_vis.integration_time))
#print("nvis, size", block_vis.nvis, block_vis.size())
gt = create_gaintable_from_blockvisibility(block_vis)
#print("np.sum(gt.data): ", numpy.sum(gt.data['gain']))
gt = simulate_gaintable(gt, phase_error=1.0)
def test_export_ms(self):
if run_ms_tests == False:
return
msoutfile = arl_path("data/vis/Test_output.ms")
from astropy.coordinates import SkyCoord
from astropy import units as u
from wrappers.serial.image.operations import show_image, export_image_to_fits
from wrappers.serial.simulation.configurations import create_named_configuration
from wrappers.serial.simulation.testing_support import create_test_image
from wrappers.serial.imaging.base import create_image_from_visibility
from wrappers.serial.imaging.base import advise_wide_field
from workflows.serial.imaging.imaging_serial import invert_list_serial_workflow, predict_list_serial_workflow
from data_models.polarisation import PolarisationFrame
lowr3 = create_named_configuration('LOWBD2', rmax=750.0)
times = numpy.zeros([1])
frequency = numpy.array([1e8])
channelbandwidth = numpy.array([1e6])
phasecentre = SkyCoord(ra=+15.0 * u.deg, dec=-45.0 * u.deg, frame='icrs', equinox='J2000')
bvis = create_blockvisibility(lowr3, times, frequency, phasecentre=phasecentre,
weight=1.0, polarisation_frame=PolarisationFrame('stokesI'),
channel_bandwidth=channelbandwidth)
vt = convert_blockvisibility_to_visibility(bvis)
advice = advise_wide_field(vt, guard_band_image=3.0, delA=0.1, facets=1, wprojection_planes=1,
oversampling_synthesised_beam=4.0)
cellsize = advice['cellsize']
m31image = create_test_image(frequency=frequency, cellsize=cellsize)
nchan, npol, ny, nx = m31image.data.shape
m31image.wcs.wcs.crval[0] = vt.phasecentre.ra.deg
m31image.wcs.wcs.crval[1] = vt.phasecentre.dec.deg
m31image.wcs.wcs.crpix[0] = float(nx // 2)
m31image.wcs.wcs.crpix[1] = float(ny // 2)
vt = predict_list_serial_workflow([vt], [m31image], context='2d')[0]
# uvdist = numpy.sqrt(vt.data['uvw'][:, 0] ** 2 + vt.data['uvw'][:, 1] ** 2)
#
# model = create_image_from_visibility(vt, cellsize=cellsize, npixel=512)
# dirty, sumwt = invert_list_serial_workflow([vt], [model], context='2d')[0]
# psf, sumwt = invert_list_serial_workflow([vt], [model], context='2d', dopsf=True)[0]
#
# show_image(dirty)
# print("Max, min in dirty image = %.6f, %.6f, sumwt = %f" % (dirty.data.max(), dirty.data.min(), sumwt))
#
# print("Max, min in PSF = %.6f, %.6f, sumwt = %f" % (psf.data.max(), psf.data.min(), sumwt))
# results_dir="/Users/f.wang"
# export_image_to_fits(dirty, '%s/imaging_dirty.fits' % (results_dir))
# export_image_to_fits(psf, '%s/imaging_psf.fits' % (results_dir))
v = convert_visibility_to_blockvisibility(vt)
vis_list=[]
vis_list.append(v)
export_blockvisility_to_ms(msoutfile, vis_list,source_name='M31')
