def deconvolve_list_channel_rsexecute_workflow(dirty_list, psf_list, model_imagelist, subimages, **kwargs):
"""Create a graph for deconvolution by channels, adding to the model
Does deconvolution channel by channel.
:param dirty_list: list or graph of dirty images
:param psf_list: list or graph of psf images. The psfs must be the size of a facet
:param model_imagelist: list of graph of models
:param subimages: Number of channels to split into
:param kwargs: Parameters for functions in components
:return:
"""
def deconvolve_subimage(dirty, psf):
assert isinstance(dirty, Image)
assert isinstance(psf, Image)
comp = deconvolve_cube(dirty, psf, **kwargs)
return comp[0]
def add_model(sum_model, model):
assert isinstance(output, Image)
assert isinstance(model, Image)
sum_model.data += model.data
return sum_model
output = rsexecute.execute(create_empty_image_like, nout=1, pure=True)(model_imagelist)
dirty_lists = rsexecute.execute(image_scatter_channels, nout=subimages, pure=True)(dirty_list[0],
subimages=subimages)
results = [rsexecute.execute(deconvolve_subimage)(dirty_list, psf_list[0])
for dirty_list in dirty_lists]
result = rsexecute.execute(image_gather_channels, nout=1, pure=True)(results, output, subimages=subimages)
result = rsexecute.execute(add_model, nout=1, pure=True)(result, model_imagelist)
return rsexecute.optimize(result)
def predict_fft_image_rsexecute_workflow(sub_bvis_list,
sub_model_list,
vp_list,
context='2d',
**kwargs):
"""Calculate residual image corresponding to a set of voltage patternss
:param sub_bvis_list: List of vis (or graph)
:param sub_model_list: List of models (or graph)
:param vp_list: List of voltage patterns (or graph)
:param context: Imaging context e.g. '2d' or 'ng'
:return:
"""
fft_model_list = [
rsexecute.execute(apply_voltage_pattern_to_image)(m, vp_list[im])
for im, m in enumerate(sub_model_list)
]
fft_bvis_list = [
rsexecute.execute(copy_visibility, nout=1)(bvis, zero=True)
for bvis in sub_bvis_list
]
fft_bvis_list = predict_list_rsexecute_workflow(fft_bvis_list,
fft_model_list,
context=context,
**kwargs)
return fft_bvis_list
def invert_skymodel_list_rsexecute_workflow(vis_list,
skymodel_list,
context,
vis_slices=1,
facets=1,
gcfcf=None,
docal=False,
**kwargs):
"""Calibrate and invert from a skymodel, iterating over the skymodel
The visibility and image are scattered, the visibility is predicted and calibrated on each part, and then the
parts are assembled. The mask if present, is multiplied in at the end.
:param vis_list: List of Visibility data models
:param skymodel_list: skymodel list
:param vis_slices: Number of vis slices (w stack or timeslice)
:param facets: Number of facets (per axis)
:param context: Type of processing e.g. 2d, wstack, timeslice or facets
:param gcfcg: tuple containing grid correction and convolution function
:param docal: Apply calibration table in skymodel
:param kwargs: Parameters for functions in components
:return: List of (image, weight) tuples)
"""
def ift_ical_sm(v, sm, g):
assert isinstance(v, Visibility) or isinstance(v, BlockVisibility), v
assert isinstance(sm, SkyModel), sm
if g is not None:
assert len(g) == 2, g
assert isinstance(g[0], Image), g[0]
assert isinstance(g[1], ConvolutionFunction), g[1]
if docal and isinstance(sm.gaintable, GainTable):
if isinstance(v, Visibility):
bv = convert_visibility_to_blockvisibility(v)
bv = apply_gaintable(bv, sm.gaintable)
v = convert_blockvisibility_to_visibility(bv)
else:
v = apply_gaintable(v, sm.gaintable)
result = invert_list_serial_workflow([v], [sm.image],
context=context,
vis_slices=vis_slices,
facets=facets,
gcfcf=[g],
**kwargs)[0]
if isinstance(sm.mask, Image):
result[0].data *= sm.mask.data
return result
if gcfcf is None:
return [
rsexecute.execute(ift_ical_sm, nout=1)(vis_list[i], sm, None)
for i, sm in enumerate(skymodel_list)
]
else:
return [
rsexecute.execute(ift_ical_sm, nout=1)(vis_list[i], sm, gcfcf[i])
for i, sm in enumerate(skymodel_list)
]
def actualSetUp(self, nfreqwin=3, dospectral=True, dopol=False,
amp_errors=None, phase_errors=None, zerow=True):
if amp_errors is None:
amp_errors = {'T': 0.0, 'G': 0.1}
if phase_errors is None:
phase_errors = {'T': 1.0, 'G': 0.0}
self.npixel = 512
self.low = create_named_configuration('LOWBD2', rmax=750.0)
self.freqwin = nfreqwin
self.vis_list = list()
self.ntimes = 1
self.times = numpy.linspace(-3.0, +3.0, self.ntimes) * numpy.pi / 12.0
self.frequency = numpy.linspace(0.8e8, 1.2e8, self.freqwin)
if self.freqwin > 1:
self.channelwidth = numpy.array(self.freqwin * [self.frequency[1] - self.frequency[0]])
else:
self.channelwidth = numpy.array([1e6])
if dopol:
self.vis_pol = PolarisationFrame('linear')
self.image_pol = PolarisationFrame('stokesIQUV')
f = numpy.array([100.0, 20.0, -10.0, 1.0])
else:
self.vis_pol = PolarisationFrame('stokesI')
self.image_pol = PolarisationFrame('stokesI')
f = numpy.array([100.0])
if dospectral:
flux = numpy.array([f * numpy.power(freq / 1e8, -0.7) for freq in self.frequency])
else:
flux = numpy.array([f])
self.phasecentre = SkyCoord(ra=+180.0 * u.deg, dec=-60.0 * u.deg, frame='icrs', equinox='J2000')
self.blockvis_list = \
[rsexecute.execute(ingest_unittest_visibility, nout=1)(self.low,
[self.frequency[i]],
[self.channelwidth[i]],
self.times,
self.vis_pol,
self.phasecentre, block=True,
zerow=zerow)
for i in range(nfreqwin)]
self.blockvis_list = rsexecute.compute(self.blockvis_list, sync=True)
for v in self.blockvis_list:
v.data['vis'][...] = 1.0 + 0.0j
self.error_blockvis_list = [rsexecute.execute(copy_visibility(v)) for v in self.blockvis_list]
gt = rsexecute.execute(create_gaintable_from_blockvisibility)(self.blockvis_list[0])
gt = rsexecute.execute(simulate_gaintable)\
(gt, phase_error=0.1, amplitude_error=0.0, smooth_channels=1, leakage=0.0, seed=180555)
self.error_blockvis_list = [rsexecute.execute(apply_gaintable)(self.error_blockvis_list[i], gt)
for i in range(self.freqwin)]
self.error_blockvis_list = rsexecute.compute(self.error_blockvis_list, sync=True)
assert numpy.max(numpy.abs(self.error_blockvis_list[0].vis - self.blockvis_list[0].vis)) > 0.0
def sum_images_rsexecute(image_list, split=2):
""" Sum a set of images, using a tree reduction
:param image_list: List of (image, sum weights) tuples
:param split: Order of split i.e. 2 is binary
:return: graph for summed (image, sumwt)
For example, to create a list of (dirty image, sumwt) tuples and then sum all::
rsexecute.set_client(use_dask=True)
dirty_list = invert_list_rsexecute_workflow(vis_list,
template_model_imagelist=model_list, context='wstack', vis_slices=51)
dirty_list = sum_image_rsexecute(dirty_list)
dirty, sumwt = rsexecute.compute(dirty_list, sync=True)
"""
def sum_images(imagelist):
out = copy_image(imagelist[0])
out.data += imagelist[1].data
return out
if len(image_list) > split:
centre = len(image_list) // split
result = [
sum_images_rsexecute(image_list[:centre]),
sum_images_rsexecute(image_list[centre:])
]
return rsexecute.execute(sum_images, nout=2)(result)
else:
return rsexecute.execute(sum_images, nout=2)(image_list)
def actualSetUp(self, freqwin=1, block=True, dopol=False, zerow=False):
self.npixel = 1024
self.low = create_named_configuration('LOWBD2', rmax=550.0)
self.freqwin = freqwin
self.blockvis_list = list()
self.ntimes = 5
self.cellsize = 0.0005
# Choose the interval so that the maximum change in w is smallish
integration_time = numpy.pi * (24 / (12 * 60))
self.times = numpy.linspace(-integration_time * (self.ntimes // 2), integration_time * (self.ntimes // 2),
self.ntimes)
if freqwin > 1:
self.frequency = numpy.linspace(0.8e8, 1.2e8, self.freqwin)
self.channelwidth = numpy.array(freqwin * [self.frequency[1] - self.frequency[0]])
else:
self.frequency = numpy.array([1.0e8])
self.channelwidth = numpy.array([4e7])
if dopol:
self.vis_pol = PolarisationFrame('linear')
self.image_pol = PolarisationFrame('stokesIQUV')
f = numpy.array([100.0, 20.0, -10.0, 1.0])
else:
self.vis_pol = PolarisationFrame('stokesI')
self.image_pol = PolarisationFrame('stokesI')
f = numpy.array([100.0])
self.phasecentre = SkyCoord(ra=+0.0 * u.deg, dec=-40.0 * u.deg, frame='icrs', equinox='J2000')
self.blockvis_list = [rsexecute.execute(ingest_unittest_visibility)(self.low,
[self.frequency[freqwin]],
[self.channelwidth[freqwin]],
self.times,
self.vis_pol,
self.phasecentre, block=block,
zerow=zerow)
for freqwin, _ in enumerate(self.frequency)]
self.blockvis_list = rsexecute.compute(self.blockvis_list, sync=True)
self.vis_list = [rsexecute.execute(convert_blockvisibility_to_visibility)(bv) for bv in self.blockvis_list]
self.vis_list = rsexecute.compute(self.vis_list, sync=True)
self.skymodel_list = [rsexecute.execute(create_low_test_skymodel_from_gleam)
(npixel=self.npixel, cellsize=self.cellsize, frequency=[self.frequency[f]],
phasecentre=self.phasecentre,
polarisation_frame=PolarisationFrame("stokesI"),
flux_limit=0.6,
flux_threshold=1.0,
flux_max=5.0) for f, freq in enumerate(self.frequency)]
self.skymodel_list = rsexecute.compute(self.skymodel_list, sync=True)
assert isinstance(self.skymodel_list[0].image, Image), self.skymodel_list[0].image
assert isinstance(self.skymodel_list[0].components[0], Skycomponent), self.skymodel_list[0].components[0]
assert len(self.skymodel_list[0].components) == 35, len(self.skymodel_list[0].components)
self.skymodel_list = expand_skymodel_by_skycomponents(self.skymodel_list[0])
assert len(self.skymodel_list) == 36, len(self.skymodel_list)
assert numpy.max(numpy.abs(self.skymodel_list[-1].image.data)) > 0.0, "Image is empty"
self.vis_list = [copy_visibility(self.vis_list[0], zero=True) for i, _ in enumerate(self.skymodel_list)]
def create_atmospheric_errors_gaintable_rsexecute_workflow(
sub_bvis_list,
sub_components,
r0=5e3,
screen=None,
height=3e5,
type_atmosphere='iono',
show=False,
basename='',
**kwargs):
""" Create gaintable for atmospheric errors
:param sub_bvis_list: List of vis (or graph)
:param sub_components: List of components (or graph)
:param r0: r0 in m
:param screen:
:param height: Height (in m) of screen above telescope e.g. 3e5
:param type_atmosphere: 'ionosphere' or 'troposhere'
:param show: Plot the results
:param basename: Base name for the plots
:return: (list of error-free gaintables, list of error gaintables) or graph
"""
# One pointing table per visibility
error_gt_list = [
rsexecute.execute(create_gaintable_from_screen)(
vis,
sub_components,
r0=r0,
screen=screen,
height=height,
type_atmosphere=type_atmosphere)
for ivis, vis in enumerate(sub_bvis_list)
]
# Create the gain tables, one per Visibility and per component
no_error_gt_list = [[
rsexecute.execute(create_gaintable_from_blockvisibility)(bvis,
**kwargs)
for cmp in sub_components
] for ibv, bvis in enumerate(sub_bvis_list)]
if show:
tmp_gt_list = rsexecute.compute(error_gt_list, sync=True)
plot_file = 'gaintable_%s.png' % r0
plot_gaintable(tmp_gt_list,
title="%s: dish 0 gain phase, %s" % (basename, r0),
value='phase',
plot_file=plot_file)
return no_error_gt_list, error_gt_list
def sum_invert_results_rsexecute(image_list, split=2):
""" Sum a set of invert results with appropriate weighting
:param image_list: List of (image, sum weights) tuples
:param split: Split into
:return: image, sum of weights
"""
if len(image_list) > split:
centre = len(image_list) // split
result = [sum_invert_results_rsexecute(image_list[:centre])]
result.append(sum_invert_results_rsexecute(image_list[centre:]))
return rsexecute.execute(sum_invert_results, nout=2)(result)
else:
return rsexecute.execute(sum_invert_results, nout=2)(image_list)
def sum_predict_results_rsexecute(bvis_list, split=2):
""" Sum a set of predict results
:param bvis_list: List of (image, sum weights) tuples
:param split: Split into
:return: BlockVis
"""
if len(bvis_list) > split:
centre = len(bvis_list) // split
result = [
sum_predict_results_rsexecute(bvis_list[:centre]),
sum_predict_results_rsexecute(bvis_list[centre:])
]
return rsexecute.execute(sum_predict_results, nout=2)(result)
else:
return rsexecute.execute(sum_predict_results, nout=2)(bvis_list)
def crosssubtract_datamodels_skymodel_list_rsexecute_workflow(
obsvis, modelvis_list):
"""Form data models by subtracting sum from the observed and adding back each model in turn
vmodel[p] = vobs - sum(i!=p) modelvis[i]
This is the E step in the Expectation-Maximisation algorithm.
:param obsvis: "Observed" visibility
:param modelvis_list: List of Visibility data model predictions
:return: List of (image, weight) tuples)
"""
# Now do the meaty part. We probably want to refactor this for performance once it works.
def vsum(ov, mv):
# Observed vis minus the sum of all predictions
verr = copy_visibility(ov)
for m in mv:
verr.data['vis'] -= m.data['vis']
# Now add back each model in turn
result = list()
for m in mv:
vr = copy_visibility(verr)
vr.data['vis'] += m.data['vis']
result.append(vr)
assert len(result) == len(mv)
return result
return rsexecute.execute(vsum, nout=len(modelvis_list))(obsvis,
modelvis_list)
def calculate_residual_from_gaintables_rsexecute_workflow(
bvis_list, components, model_list, no_error_gtl, error_gtl, **kwargs):
""" Calculate residual between two gaintables
:param bvis_list:
:param components:
:param model_list:
:param no_error_gtl:
:param error_gtl:
:param kwargs:
:return:
"""
error_dirty_list = \
calculate_residual_dft_rsexecute_workflow(bvis_list, components, model_list, error_gtl)
no_error_dirty_list = \
calculate_residual_dft_rsexecute_workflow(bvis_list, components, model_list, no_error_gtl)
def subtract(im1, im2):
im = copy_image(im1[0])
im.data -= im2[0].data
return im, im1[1]
residual_list = rsexecute.execute(subtract, nout=1)(error_dirty_list,
no_error_dirty_list)
return residual_list
def test_zero_list(self):
self.actualSetUp()
centre = self.freqwin // 2
vis_list = zero_list_rsexecute_workflow(self.bvis_list)
vis_list = rsexecute.compute(vis_list, sync=True)
assert numpy.max(numpy.abs(vis_list[centre].vis)) < 1e-15, numpy.max(
numpy.abs(vis_list[centre].vis))
predicted_vis_list = [
rsexecute.execute(dft_skycomponent_visibility)(
vis_list[freqwin], self.components_list[freqwin])
for freqwin, _ in enumerate(self.frequency)
]
predicted_vis_list = rsexecute.compute(predicted_vis_list, sync=True)
assert numpy.max(numpy.abs(predicted_vis_list[centre].vis)) > 0.0, \
numpy.max(numpy.abs(predicted_vis_list[centre].vis))
diff_vis_list = subtract_list_rsexecute_workflow(
self.bvis_list, predicted_vis_list)
diff_vis_list = rsexecute.compute(diff_vis_list, sync=True)
assert numpy.max(numpy.abs(
diff_vis_list[centre].vis)) < 1e-15, numpy.max(
numpy.abs(diff_vis_list[centre].vis))
def test_predict(self):
self.actualSetUp(zerow=True)
self.skymodel_list = [
rsexecute.execute(create_low_test_skymodel_from_gleam)(
npixel=self.npixel,
cellsize=self.cellsize,
frequency=[self.frequency[f]],
phasecentre=self.phasecentre,
polarisation_frame=PolarisationFrame("stokesI"),
flux_limit=0.3,
flux_threshold=1.0,
flux_max=5.0) for f, freq in enumerate(self.frequency)
]
self.skymodel_list = rsexecute.compute(self.skymodel_list, sync=True)
assert isinstance(self.skymodel_list[0].image,
Image), self.skymodel_list[0].image
assert isinstance(self.skymodel_list[0].components[0],
Skycomponent), self.skymodel_list[0].components[0]
assert len(self.skymodel_list[0].components) == 25, len(
self.skymodel_list[0].components)
assert numpy.max(numpy.abs(
self.skymodel_list[0].image.data)) > 0.0, "Image is empty"
self.skymodel_list = rsexecute.scatter(self.skymodel_list)
skymodel_vislist = predict_skymodel_list_rsexecute_workflow(
self.vis_list[0], self.skymodel_list, context='2d')
skymodel_vislist = rsexecute.compute(skymodel_vislist, sync=True)
assert numpy.max(numpy.abs(skymodel_vislist[0].vis)) > 0.0
def test_useDaskSync(self):
def square(x):
return x**2
graph = rsexecute.execute(square)(numpy.arange(10))
result = rsexecute.compute(graph, sync=True)
assert (result == numpy.array([0, 1, 4, 9, 16, 25, 36, 49, 64,
81])).all()
def restore_list_rsexecute_workflow(model_imagelist, psf_imagelist, residual_imagelist=None, restore_facets=1,
restore_overlap=0, restore_taper='tukey', **kwargs):
""" Create a graph to calculate the restored image
:param model_imagelist: Model list
:param psf_imagelist: PSF list
:param residual_imagelist: Residual list
:param kwargs: Parameters for functions in components
:param restore_facets: Number of facets used per axis (used to distribute)
:param restore_overlap: Overlap in pixels (0 is best)
:param restore_taper: Type of taper.
:return:
"""
assert len(model_imagelist) == len(psf_imagelist)
if residual_imagelist is not None:
assert len(model_imagelist) == len(residual_imagelist)
if restore_facets % 2 == 0 or restore_facets == 1:
actual_number_facets = restore_facets
else:
actual_number_facets = max(1, (restore_facets - 1))
psf_list = rsexecute.execute(remove_sumwt, nout=len(psf_imagelist))(psf_imagelist)
# Scatter each list element into a list. We will then run restore_cube on each
facet_model_list = [rsexecute.execute(image_scatter_facets, nout=actual_number_facets * actual_number_facets)
(model, facets=restore_facets, overlap=restore_overlap, taper=restore_taper)
for model in model_imagelist]
facet_psf_list = [rsexecute.execute(image_scatter_facets, nout=actual_number_facets * actual_number_facets)
(psf, facets=restore_facets, overlap=restore_overlap, taper=restore_taper)
for psf in psf_list]
if residual_imagelist is not None:
residual_list = rsexecute.execute(remove_sumwt, nout=len(residual_imagelist))(residual_imagelist)
facet_residual_list = [
rsexecute.execute(image_scatter_facets, nout=actual_number_facets * actual_number_facets)
(residual, facets=restore_facets, overlap=restore_overlap, taper=restore_taper)
for residual in residual_list]
facet_restored_list = [[rsexecute.execute(restore_cube, nout=actual_number_facets * actual_number_facets)
(model=facet_model_list[i][im], psf=facet_psf_list[i][im],
residual=facet_residual_list[i][im],
**kwargs)
for im, _ in enumerate(facet_model_list[i])] for i, _ in enumerate(model_imagelist)]
else:
facet_restored_list = [[rsexecute.execute(restore_cube, nout=actual_number_facets * actual_number_facets)
(model=facet_model_list[i][im], psf=facet_psf_list[i][im],
**kwargs)
for im, _ in enumerate(facet_model_list[i])] for i, _ in enumerate(model_imagelist)]
# Now we run restore_cube on each and gather the results across all facets
restored_imagelist = [rsexecute.execute(image_gather_facets)
(facet_restored_list[i], model_imagelist[i], facets=restore_facets,
overlap=restore_overlap, taper=restore_taper)
for i, _ in enumerate(model_imagelist)]
return rsexecute.optimize(restored_imagelist)
def predict_dft_rsexecute_workflow(sub_bvis_list,
sub_components,
gt_list,
context='2d',
**kwargs):
"""Calculate residual image corresponding to a set of gaintables
The visibility difference for a set of components for error and no error gaintables
are calculated and the residual images constructed
:param sum_vis:
:param sub_bvis_list: List of vis (or graph)
:param sub_components: List of components (or graph)
:param sub_model_list: List of models (or graph)
:param no_error_gt_list: List of gaintables for no error (or graph)
:param context: Imaging context e.g. '2d' or 'ng'
:param residual: Calculate residual visibility (True)
:return:
"""
dft_sm_list = [[
rsexecute.execute(SkyModel, nout=1)(components=[sub_components[icomp]],
gaintable=gt_list[ibv][icomp])
for icomp, _ in enumerate(sub_components)
] for ibv, bv in enumerate(sub_bvis_list)]
# Predict each visibility for each skymodel. We keep all the visibilities separate
# and add up dirty images at the end of processing. We calibrate which applies the voltage pattern
dft_bvis_list = [
rsexecute.execute(copy_visibility, nout=1)(bvis, zero=True)
for bvis in sub_bvis_list
]
dft_bvis_list = [
predict_skymodel_list_compsonly_rsexecute_workflow(dft_bvis_list[ibv],
dft_sm_list[ibv],
context=context,
docal=True)
for ibv, bvis in enumerate(dft_bvis_list)
]
dft_bvis_list = \
[sum_predict_results_rsexecute([dft_bvis_list[ivis][icomp]
for icomp, _ in enumerate(sub_components)])
for ivis, _ in enumerate(dft_bvis_list)]
return dft_bvis_list
def taper_list_rsexecute_workflow(vis_list, size_required):
"""Taper to desired size
:param vis_list:
:param size_required:
:return:
"""
result = [rsexecute.execute(taper_visibility_gaussian, nout=1)(v, beam=size_required) for v in vis_list]
return rsexecute.optimize(result)
def image_rsexecute_map_workflow(im,
imfunction,
facets=1,
overlap=0,
taper=None,
**kwargs):
"""Apply a function across an image: scattering to subimages, applying the function, and then gathering
:param im: Image to be processed
:param imfunction: Function to be applied
:param facets: See image_scatter_facets
:param overlap: image_scatter_facets
:param taper: image_scatter_facets
:param kwargs: kwargs for imfunction
:return: graph for output image
For example::
rsexecute.set_client(use_dask=True)
model = create_test_image(frequency=frequency, phasecentre=phasecentre, cellsize=0.001,
polarisation_frame=PolarisationFrame('stokesI'))
def imagerooter(im, **kwargs):
im.data = numpy.sqrt(numpy.abs(im.data))
return im
root_graph = image_rsexecute_map_workflow(model, imagerooter, facets=16)
root_image = rsexecute.compute(root_graph, sync=True)
"""
facets_list = rsexecute.execute(image_scatter_facets,
nout=facets**2)(im,
facets=facets,
overlap=overlap,
taper=taper)
root_list = [
rsexecute.execute(imfunction)(facet, **kwargs) for facet in facets_list
]
gathered = rsexecute.execute(image_gather_facets)(root_list,
im,
facets=facets,
overlap=overlap,
taper=taper)
return gathered
def predict_fft_components_rsexecute_workflow(sub_bvis_list,
sub_components,
sub_model_list,
vp_list,
context='2d',
insert_method='Lanczos',
**kwargs):
"""Calculate residual image corresponding to a set of voltage patternss
:param sum_vis:
:param sub_bvis_list: List of vis (or graph)
:param sub_components: List of components (or graph)
:param sub_model_list: List of models (or graph)
:param vp_list: List of gaintables for no error (or graph)
:param context: Imaging context e.g. '2d' or 'ng'
:param residual: Calculate residual visibility (True)
:return:
"""
fft_model_list = [
rsexecute.execute(create_empty_image_like)(m) for m in sub_model_list
]
fft_model_list = [
rsexecute.execute(insert_skycomponent)(m,
sc=sub_components,
insert_method=insert_method)
for m in fft_model_list
]
fft_model_list = [
rsexecute.execute(apply_voltage_pattern_to_image)(m, vp_list[im])
for im, m in enumerate(fft_model_list)
]
fft_bvis_list = [
rsexecute.execute(copy_visibility, nout=1)(bvis, zero=True)
for bvis in sub_bvis_list
]
fft_bvis_list = predict_list_rsexecute_workflow(fft_bvis_list,
fft_model_list,
context=context,
**kwargs)
return fft_bvis_list
def corrupt_list_rsexecute_workflow(vis_list,
gt_list=None,
seed=None,
**kwargs):
""" Create a graph to apply gain errors to a vis_list
:param vis_list: List of vis (or graph)
:param gt_list: Optional gain table graph
:param kwargs:
:return: list of vis (or graph)
"""
def corrupt_vis(vis, gt, **kwargs):
if isinstance(vis, Visibility):
bv = convert_visibility_to_blockvisibility(vis)
else:
bv = vis
if gt is None:
gt = create_gaintable_from_blockvisibility(bv, **kwargs)
gt = simulate_gaintable(gt, **kwargs)
bv = apply_gaintable(bv, gt)
if isinstance(vis, Visibility):
return convert_blockvisibility_to_visibility(bv)
else:
return bv
if gt_list is None:
return [
rsexecute.execute(corrupt_vis, nout=1)(vis_list[ivis], None,
**kwargs)
for ivis, v in enumerate(vis_list)
]
else:
return [
rsexecute.execute(corrupt_vis, nout=1)(vis_list[ivis],
gt_list[ivis], **kwargs)
for ivis, v in enumerate(vis_list)
]
def zero_list_rsexecute_workflow(vis_list):
""" Creates a new vis_list and initialises all to zero
:param vis_list: List of vis (or graph)
:return: List of vis (or graph)
"""
def zero(vis):
if vis is not None:
zerovis = copy_visibility(vis, zero=True)
return zerovis
else:
return None
result = [rsexecute.execute(zero, pure=True, nout=1)(v) for v in vis_list]
return rsexecute.optimize(result)
def actualSetUp(self, freqwin=1, block=False, dopol=False, zerow=False):
self.npixel = 1024
self.low = create_named_configuration('LOWBD2', rmax=750.0)
self.freqwin = freqwin
self.vis_list = list()
self.ntimes = 5
self.cellsize = 0.0005
# Choose the interval so that the maximum change in w is smallish
integration_time = numpy.pi * (24 / (12 * 60))
self.times = numpy.linspace(-integration_time * (self.ntimes // 2),
integration_time * (self.ntimes // 2),
self.ntimes)
if freqwin > 1:
self.frequency = numpy.linspace(0.8e8, 1.2e8, self.freqwin)
self.channelwidth = numpy.array(
freqwin * [self.frequency[1] - self.frequency[0]])
else:
self.frequency = numpy.array([1.0e8])
self.channelwidth = numpy.array([4e7])
if dopol:
self.vis_pol = PolarisationFrame('linear')
self.image_pol = PolarisationFrame('stokesIQUV')
f = numpy.array([100.0, 20.0, -10.0, 1.0])
else:
self.vis_pol = PolarisationFrame('stokesI')
self.image_pol = PolarisationFrame('stokesI')
f = numpy.array([100.0])
self.phasecentre = SkyCoord(ra=+30.0 * u.deg,
dec=-60.0 * u.deg,
frame='icrs',
equinox='J2000')
self.vis_list = [
rsexecute.execute(ingest_unittest_visibility)(
self.low, [self.frequency[freqwin]],
[self.channelwidth[freqwin]],
self.times,
self.vis_pol,
self.phasecentre,
block=block,
zerow=zerow) for freqwin, _ in enumerate(self.frequency)
]
self.vis_list = rsexecute.compute(self.vis_list)
def zero_list_rsexecute_workflow(vis_list):
""" Initialise vis to zero: creates new data holders
:param vis_list:
:return: List of vis_lists
"""
def zero(vis):
if vis is not None:
zerovis = copy_visibility(vis)
zerovis.data['vis'][...] = 0.0
return zerovis
else:
return None
result = [rsexecute.execute(zero, pure=True, nout=1)(v) for v in vis_list]
return rsexecute.optimize(result)
def continuum_imaging_list_rsexecute_workflow(vis_list, model_imagelist, context, gcfcf=None,
vis_slices=1, facets=1, **kwargs):
""" Create graph for the continuum imaging pipeline.
Same as ICAL but with no selfcal.
:param vis_list:
:param model_imagelist:
:param context: Imaging context
:param kwargs: Parameters for functions in components
:return:
"""
if gcfcf is None:
gcfcf = [rsexecute.execute(create_pswf_convolutionfunction)(model_imagelist[0])]
psf_imagelist = invert_list_rsexecute_workflow(vis_list, model_imagelist, context=context, dopsf=True,
vis_slices=vis_slices, facets=facets, gcfcf=gcfcf, **kwargs)
residual_imagelist = residual_list_rsexecute_workflow(vis_list, model_imagelist, context=context, gcfcf=gcfcf,
vis_slices=vis_slices, facets=facets, **kwargs)
deconvolve_model_imagelist = deconvolve_list_rsexecute_workflow(residual_imagelist, psf_imagelist,
model_imagelist,
prefix='cycle 0',
**kwargs)
nmajor = get_parameter(kwargs, "nmajor", 5)
if nmajor > 1:
for cycle in range(nmajor):
prefix = "cycle %d" % (cycle + 1)
residual_imagelist = residual_list_rsexecute_workflow(vis_list, deconvolve_model_imagelist,
context=context, vis_slices=vis_slices,
facets=facets,
gcfcf=gcfcf, **kwargs)
deconvolve_model_imagelist = deconvolve_list_rsexecute_workflow(residual_imagelist, psf_imagelist,
deconvolve_model_imagelist,
prefix=prefix,
**kwargs)
residual_imagelist = residual_list_rsexecute_workflow(vis_list, deconvolve_model_imagelist, context=context,
vis_slices=vis_slices, facets=facets, gcfcf=gcfcf, **kwargs)
restore_imagelist = restore_list_rsexecute_workflow(deconvolve_model_imagelist, psf_imagelist, residual_imagelist)
return (deconvolve_model_imagelist, residual_imagelist, restore_imagelist)
def subtract_list_rsexecute_workflow(vis_list, model_vislist):
""" Initialise vis to zero
:param vis_list:
:param model_vislist: Model to be subtracted
:return: List of vis_lists
"""
def subtract_vis(vis, model_vis):
if vis is not None and model_vis is not None:
assert vis.vis.shape == model_vis.vis.shape
subvis = copy_visibility(vis)
subvis.data['vis'][...] -= model_vis.data['vis'][...]
return subvis
else:
return None
result = [rsexecute.execute(subtract_vis, pure=True, nout=1)(vis=vis_list[i],
model_vis=model_vislist[i])
for i in range(len(vis_list))]
return rsexecute.optimize(result)
def predict_skymodel_list_compsonly_rsexecute_workflow(obsvis,
skymodel_list,
docal=False,
**kwargs):
"""Predict from a list of component-only skymodels, producing one visibility per skymodel
This is an optimised version of predict_skymodel_list_rsexecute_workflow, working on block
visibilities and ignoring the image in a skymodel
:param obsvis: "Observed Block Visibility"
:param skymodel_list: skymodel list
:param context: Type of processing e.g. 2d, wstack, timeslice or facets
:param docal: Apply calibration table in skymodel
:param kwargs: Parameters for functions in components
:return: List of vis_lists
"""
def ft_cal_sm(obv, sm):
assert isinstance(obv, BlockVisibility), obv
bv = copy_visibility(obv)
bv.data['vis'][...] = 0.0 + 0.0j
assert len(sm.components) > 0
if isinstance(sm.mask, Image):
comps = copy_skycomponent(sm.components)
comps = apply_beam_to_skycomponent(comps, sm.mask)
bv = predict_skycomponent_visibility(bv, comps)
else:
bv = predict_skycomponent_visibility(bv, sm.components)
if docal and isinstance(sm.gaintable, GainTable):
bv = apply_gaintable(bv, sm.gaintable, inverse=True)
return bv
return [
rsexecute.execute(ft_cal_sm, nout=1)(obsvis, sm)
for sm in skymodel_list
]
# Define a function to be executed by Dask to load the data, combine it, and select
# only the short baselines. We load each channel separately.
def load_ms(c):
v1 = create_visibility_from_ms(input_vis[0], start_chan=c,
end_chan=c)[0]
v2 = create_visibility_from_ms(input_vis[1], start_chan=c,
end_chan=c)[0]
vf = append_visibility(v1, v2)
vf = convert_visibility_to_stokes(vf)
vf.configuration.diameter[...] = 35.0
rows = vis_select_uvrange(vf, 0.0, uvmax=uvmax)
return create_visibility_from_rows(vf, rows)
# Construct the graph to load the data and persist the graph on the Dask cluster.
vis_list = [rsexecute.execute(load_ms)(c) for c in range(nchan)]
vis_list = rsexecute.persist(vis_list)
# Construct the graph to define the model images and persist the graph to the cluster
model_list = [
rsexecute.execute(create_image_from_visibility)(
v,
npixel=npixel,
cellsize=cellsize,
polarisation_frame=PolarisationFrame("stokesIQUV"),
nchan=1) for v in vis_list
]
model_list = rsexecute.persist(model_list)
# Construct the graphs to make the dirty image and psf, and persist these to the cluster
dirty_list = invert_list_rsexecute_workflow(
def deconvolve_list_rsexecute_workflow(dirty_list, psf_list, model_imagelist, prefix='', mask=None, **kwargs):
"""Create a graph for deconvolution, adding to the model
Note that this call can be converted to a set of rsexecute calls to the serial
version, using argument use_serial_clean=True
:param dirty_list: list of dirty images (or graph)
:param psf_list: list of psfs (or graph)
:param model_imagelist: list of models (or graph)
:param prefix: Informative prefix to log messages
:param mask: Mask for deconvolution
:param kwargs: Parameters for functions
:return: graph for the deconvolution
For example::
dirty_imagelist = invert_list_rsexecute_workflow(vis_list, model_imagelist, context='2d',
dopsf=False, normalize=True)
psf_imagelist = invert_list_rsexecute_workflow(vis_list, model_imagelist, context='2d',
dopsf=True, normalize=True)
dirty_imagelist = rsexecute.persist(dirty_imagelist)
psf_imagelist = rsexecute.persist(psf_imagelist)
dec_imagelist = deconvolve_list_rsexecute_workflow(dirty_imagelist, psf_imagelist,
model_imagelist, niter=1000, fractional_threshold=0.01,
scales=[0, 3, 10], algorithm='mmclean', nmoment=3, nchan=freqwin,
threshold=0.1, gain=0.7)
dec_imagelist = rsexecute.persist(dec_imagelist)
"""
nchan = len(dirty_list)
# Number of moments. 1 is the sum.
nmoment = get_parameter(kwargs, "nmoment", 1)
if get_parameter(kwargs, "use_serial_clean", False):
from rascil.workflows.serial.imaging.imaging_serial import deconvolve_list_serial_workflow
return rsexecute.execute(deconvolve_list_serial_workflow, nout=nchan) \
(dirty_list, psf_list, model_imagelist, prefix=prefix, mask=mask, **kwargs)
def deconvolve(dirty, psf, model, facet, gthreshold, msk=None):
if prefix == '':
lprefix = "facet %d" % facet
else:
lprefix = "%s, facet %d" % (prefix, facet)
if nmoment > 0:
moment0 = calculate_image_frequency_moments(dirty)
this_peak = numpy.max(numpy.abs(moment0.data[0, ...])) / dirty.data.shape[0]
else:
ref_chan = dirty.data.shape[0] // 2
this_peak = numpy.max(numpy.abs(dirty.data[ref_chan, ...]))
if this_peak > 1.1 * gthreshold:
kwargs['threshold'] = gthreshold
result, _ = deconvolve_cube(dirty, psf, prefix=lprefix, mask=msk, **kwargs)
if result.data.shape[0] == model.data.shape[0]:
result.data += model.data
return result
else:
return copy_image(model)
deconvolve_facets = get_parameter(kwargs, 'deconvolve_facets', 1)
deconvolve_overlap = get_parameter(kwargs, 'deconvolve_overlap', 0)
deconvolve_taper = get_parameter(kwargs, 'deconvolve_taper', None)
if deconvolve_facets > 1 and deconvolve_overlap > 0:
deconvolve_number_facets = (deconvolve_facets - 2) ** 2
else:
deconvolve_number_facets = deconvolve_facets ** 2
scattered_channels_facets_model_list = \
[rsexecute.execute(image_scatter_facets, nout=deconvolve_number_facets)(m, facets=deconvolve_facets,
overlap=deconvolve_overlap,
taper=deconvolve_taper)
for m in model_imagelist]
scattered_facets_model_list = [
rsexecute.execute(image_gather_channels, nout=1)([scattered_channels_facets_model_list[chan][facet]
for chan in range(nchan)])
for facet in range(deconvolve_number_facets)]
# Scatter the separate channel images into deconvolve facets and then gather channels for each facet.
# This avoids constructing the entire spectral cube.
# i.e. SCATTER BY FACET then SCATTER BY CHANNEL
dirty_list_trimmed = rsexecute.execute(remove_sumwt, nout=nchan)(dirty_list)
scattered_channels_facets_dirty_list = \
[rsexecute.execute(image_scatter_facets, nout=deconvolve_number_facets)(d, facets=deconvolve_facets,
overlap=deconvolve_overlap,
taper=deconvolve_taper)
for d in dirty_list_trimmed]
scattered_facets_dirty_list = [
rsexecute.execute(image_gather_channels, nout=1)([scattered_channels_facets_dirty_list[chan][facet]
for chan in range(nchan)])
for facet in range(deconvolve_number_facets)]
psf_list_trimmed = rsexecute.execute(remove_sumwt, nout=nchan)(psf_list)
def extract_psf(psf, facets):
spsf = create_empty_image_like(psf)
cx = spsf.shape[3] // 2
cy = spsf.shape[2] // 2
wx = spsf.shape[3] // facets
wy = spsf.shape[2] // facets
xbeg = cx - wx // 2
xend = cx + wx // 2
ybeg = cy - wy // 2
yend = cy + wy // 2
spsf.data = psf.data[..., ybeg:yend, xbeg:xend]
spsf.wcs.wcs.crpix[0] -= xbeg
spsf.wcs.wcs.crpix[1] -= ybeg
return spsf
psf_list_trimmed = [rsexecute.execute(extract_psf)(p, deconvolve_facets) for p in psf_list_trimmed]
psf_centre = rsexecute.execute(image_gather_channels, nout=1)([psf_list_trimmed[chan]
for chan in range(nchan)])
# Work out the threshold. Need to find global peak over all dirty_list images
threshold = get_parameter(kwargs, "threshold", 0.0)
fractional_threshold = get_parameter(kwargs, "fractional_threshold", 0.1)
nmoment = get_parameter(kwargs, "nmoment", 1)
use_moment0 = nmoment > 0
# Find the global threshold. This uses the peak in the average on the frequency axis since we
# want to use it in a stopping criterion in a moment clean
global_threshold = rsexecute.execute(threshold_list, nout=1)(scattered_facets_dirty_list, threshold,
fractional_threshold,
use_moment0=use_moment0, prefix=prefix)
facet_list = numpy.arange(deconvolve_number_facets).astype('int')
if mask is None:
scattered_results_list = [
rsexecute.execute(deconvolve, nout=1)(d, psf_centre, m, facet,
global_threshold)
for d, m, facet in zip(scattered_facets_dirty_list, scattered_facets_model_list, facet_list)]
else:
mask_list = \
rsexecute.execute(image_scatter_facets, nout=deconvolve_number_facets)(mask,
facets=deconvolve_facets,
overlap=deconvolve_overlap)
scattered_results_list = [
rsexecute.execute(deconvolve, nout=1)(d, psf_centre, m, facet,
global_threshold, msk)
for d, m, facet, msk in
zip(scattered_facets_dirty_list, scattered_facets_model_list, facet_list, mask_list)]
# We want to avoid constructing the entire cube so we do the inverse of how we got here:
# i.e. SCATTER BY CHANNEL then GATHER BY FACET
# Gather the results back into one image, correcting for overlaps as necessary. The taper function is is used to
# feather the facets together
# gathered_results_list = rsexecute.execute(image_gather_facets, nout=1)(scattered_results_list,
# deconvolve_model_imagelist,
# facets=deconvolve_facets,
# overlap=deconvolve_overlap,
# taper=deconvolve_taper)
# result_list = rsexecute.execute(image_scatter_channels, nout=nchan)(gathered_results_list, subimages=nchan)
scattered_channel_results_list = [rsexecute.execute(image_scatter_channels, nout=nchan)(scat, subimages=nchan)
for scat in scattered_results_list]
# The structure is now [[channels] for facets]. We do the reverse transpose to the one above.
result_list = [rsexecute.execute(image_gather_facets, nout=1)([scattered_channel_results_list[facet][chan]
for facet in range(deconvolve_number_facets)],
model_imagelist[chan], facets=deconvolve_facets,
overlap=deconvolve_overlap)
for chan in range(nchan)]
return rsexecute.optimize(result_list)
def predict_list_rsexecute_workflow(vis_list, model_imagelist, context, vis_slices=1, facets=1,
gcfcf=None, **kwargs):
"""Predict, iterating over both the scattered vis_list and image
The visibility and image are scattered, the visibility is predicted on each part, and then the
parts are assembled.
Note that this call can be converted to a set of rsexecute calls to the serial
version, using argument use_serial_predict=True
:param vis_list: list of vis (or graph)
:param model_imagelist: list of models (or graph)
:param vis_slices: Number of vis slices (w stack or timeslice)
:param facets: Number of facets (per axis)
:param context: Type of processing e.g. 2d, wstack, timeslice or facets
:param gcfcg: tuple containing grid correction and convolution function
:param kwargs: Parameters for functions in components
:return: List of vis_lists
For example::
dprepb_model = [rsexecute.execute(create_low_test_image_from_gleam)
(npixel=npixel, frequency=[frequency[f]], channel_bandwidth=[channel_bandwidth[f]],
cellsize=cellsize, phasecentre=phasecentre, polarisation_frame=PolarisationFrame("stokesI"),
flux_limit=3.0, applybeam=True)
for f, freq in enumerate(frequency)]
dprepb_model_list = rsexecute.persist(dprepb_model_list)
predicted_vis_list = predict_list_rsexecute_workflow(vis_list, model_imagelist=dprepb_model_list,
context='wstack', vis_slices=51)
predicted_vis_list = rsexecute.compute(predicted_vis_list , sync=True)
"""
if get_parameter(kwargs, "use_serial_predict", False):
from rascil.workflows.serial.imaging.imaging_serial import predict_list_serial_workflow
return [rsexecute.execute(predict_list_serial_workflow, nout=1) \
(vis_list=[vis_list[i]],
model_imagelist=[model_imagelist[i]], vis_slices=vis_slices,
facets=facets, context=context, gcfcf=gcfcf, **kwargs)[0]
for i, _ in enumerate(vis_list)]
# Predict_2d does not clear the vis so we have to do it here.
vis_list = zero_list_rsexecute_workflow(vis_list)
c = imaging_context(context)
vis_iter = c['vis_iterator']
predict = c['predict']
if facets % 2 == 0 or facets == 1:
actual_number_facets = facets
else:
actual_number_facets = facets - 1
def predict_ignore_none(vis, model, g):
if vis is not None:
assert isinstance(vis, Visibility) or isinstance(vis, BlockVisibility), vis
assert isinstance(model, Image), model
return predict(vis, model, context=context, gcfcf=g, **kwargs)
else:
return None
if gcfcf is None:
gcfcf = [rsexecute.execute(create_pswf_convolutionfunction)(m) for m in model_imagelist]
# Loop over all frequency windows
if facets == 1:
image_results_list = list()
for ivis, subvis in enumerate(vis_list):
if len(gcfcf) > 1:
g = gcfcf[ivis]
else:
g = gcfcf[0]
# Create the graph to divide the visibility into slices. This is by copy.
sub_vis_lists = rsexecute.execute(visibility_scatter, nout=vis_slices)(subvis,
vis_iter, vis_slices)
image_vis_lists = list()
# Loop over sub visibility
for sub_vis_list in sub_vis_lists:
# Predict visibility for this sub-visibility from this image
image_vis_list = rsexecute.execute(predict_ignore_none, pure=True, nout=1) \
(sub_vis_list, model_imagelist[ivis], g)
# Sum all sub-visibilities
image_vis_lists.append(image_vis_list)
image_results_list.append(rsexecute.execute(visibility_gather, nout=1)
(image_vis_lists, subvis, vis_iter))
result = image_results_list
else:
image_results_list_list = list()
for ivis, subvis in enumerate(vis_list):
# Create the graph to divide an image into facets. This is by reference.
facet_lists = rsexecute.execute(image_scatter_facets, nout=actual_number_facets ** 2)(
model_imagelist[ivis],
facets=facets)
# Create the graph to divide the visibility into slices. This is by copy.
sub_vis_lists = rsexecute.execute(visibility_scatter, nout=vis_slices)\
(subvis, vis_iter, vis_slices)
facet_vis_lists = list()
# Loop over sub visibility
for sub_vis_list in sub_vis_lists:
facet_vis_results = list()
# Loop over facets
for facet_list in facet_lists:
# Predict visibility for this subvisibility from this facet
facet_vis_list = rsexecute.execute(predict_ignore_none, pure=True, nout=1)\
(sub_vis_list, facet_list, None)
facet_vis_results.append(facet_vis_list)
# Sum the current sub-visibility over all facets
facet_vis_lists.append(rsexecute.execute(sum_predict_results)(facet_vis_results))
# Sum all sub-visibilities
image_results_list_list.append(
rsexecute.execute(visibility_gather, nout=1)(facet_vis_lists, subvis, vis_iter))
result = image_results_list_list
return rsexecute.optimize(result)
def weight_list_rsexecute_workflow(vis_list, model_imagelist, gcfcf=None, weighting='uniform', **kwargs):
""" Weight the visibility data
This is done collectively so the weights are summed over all vis_lists and then
corrected
:param vis_list:
:param model_imagelist: Model required to determine weighting parameters
:param weighting: Type of weighting
:param kwargs: Parameters for functions in graphs
:return: List of vis_graphs
For example::
vis_list = weight_list_rsexecute_workflow(vis_list, model_list, weighting='uniform')
"""
centre = len(model_imagelist) // 2
if gcfcf is None:
gcfcf = [rsexecute.execute(create_pswf_convolutionfunction)(model_imagelist[centre])]
def to_vis(v):
if isinstance(v, BlockVisibility):
av = convert_blockvisibility_to_visibility(v)
return av
else:
return v
avis_list = [rsexecute.execute(to_vis, nout=1)(vis) for vis in vis_list]
def grid_wt(vis, model, g):
if vis is not None:
if model is not None:
griddata = create_griddata_from_image(model)
griddata = grid_weight_to_griddata(vis, griddata, g[0][1])
return griddata
else:
return None
else:
return None
weight_list = [rsexecute.execute(grid_wt, pure=True, nout=1)(avis_list[i], model_imagelist[i],
gcfcf)
for i in range(len(vis_list))]
merged_weight_grid = rsexecute.execute(griddata_merge_weights, nout=1)(weight_list)
merged_weight_grid = rsexecute.persist(merged_weight_grid, broadcast=True)
def re_weight(vis, model, gd, g):
if gd is not None:
if vis is not None:
# Ensure that the griddata has the right axes so that the convolution
# function mapping works
agd = create_griddata_from_image(model)
agd.data = gd[0].data
vis = griddata_reweight(vis, agd, g[0][1])
return vis
else:
return None
else:
return vis
avis_list = [rsexecute.execute(re_weight, nout=1)(v, model_imagelist[i], merged_weight_grid, gcfcf)
for i, v in enumerate(avis_list)]
def to_bvis(v, ov):
if isinstance(ov, BlockVisibility):
av = convert_visibility_to_blockvisibility(v)
return av
else:
return v
result = [rsexecute.execute(to_bvis, nout=1)(vis, ovis) for vis, ovis in zip(avis_list, vis_list)]
return rsexecute.optimize(result)
