def output_skymodel(model_list, comp_list):
if conf['create_skymodel']["fill_image"]:
skymodel = SkyModel(images=model_list, components=[])
else:
skymodel = SkyModel(images=[], components=comp_list)
return BufferSkyModel(conf["buffer"], conf["outputs"]["skymodel"], skymodel).sync()
def output_skymodel(model_list, comp_list):
if conf['create_skymodel']["fill_image"]:
skymodel = SkyModel(images=model_list, components=[])
else:
skymodel = SkyModel(images=[], components=comp_list)
return memory_data_model_to_buffer(skymodel, conf["buffer"],
conf["outputs"]["skymodel"])
def convert_hdf_to_skymodel(f):
"""
:param f:
:return:
"""
assert f.attrs['ARL_data_model'] == "SkyModel", f.attrs['ARL_data_model']
fixed = f.attrs['fixed']
ncomponents = f.attrs['number_skycomponents']
components = list()
for i in range(ncomponents):
cf = f[('skycomponent%d' % i)]
components.append(convert_hdf_to_skycomponent(cf))
if 'image' in f.keys():
cf = f['image']
image = convert_hdf_to_image(cf)
else:
image = None
if 'mask' in f.keys():
cf = f['mask']
mask = convert_hdf_to_image(cf)
else:
mask = None
if 'gaintable' in f.keys():
cf = f['gaintable']
gaintable = convert_hdf_to_gaintable(cf)
else:
gaintable = None
return SkyModel(image=image, components=components, gaintable=gaintable, mask=mask, fixed=fixed)
def test_crosssubtract_datamodel(self):
self.actualSetUp(zerow=True)
future_vis = arlexecute.scatter(self.vis_list[0])
future_skymodel_list = arlexecute.scatter(self.skymodel_list)
skymodel_vislist = predict_skymodel_list_arlexecute_workflow(
future_vis, future_skymodel_list, context='2d', docal=True)
skymodel_vislist = arlexecute.compute(skymodel_vislist, sync=True)
vobs = sum_predict_results(skymodel_vislist)
future_vobs = arlexecute.scatter(vobs)
skymodel_vislist = crosssubtract_datamodels_skymodel_list_arlexecute_workflow(
future_vobs, skymodel_vislist)
skymodel_vislist = arlexecute.compute(skymodel_vislist, sync=True)
result_skymodel = [
SkyModel(components=None, image=self.skymodel_list[-1].image)
for v in skymodel_vislist
]
self.vis_list = arlexecute.scatter(self.vis_list)
result_skymodel = invert_skymodel_list_arlexecute_workflow(
skymodel_vislist, result_skymodel, context='2d', docal=True)
results = arlexecute.compute(result_skymodel, sync=True)
assert numpy.max(numpy.abs(results[0][0].data)) > 0.0
assert numpy.max(numpy.abs(results[0][1])) > 0.0
if self.plot:
import matplotlib.pyplot as plt
from wrappers.arlexecute.image.operations import show_image
show_image(results[0][0],
title='Dirty image after cross-subtraction',
vmax=0.1,
vmin=-0.01)
plt.show()
def copy_skymodel(sm):
""" Copy a sky model
"""
return SkyModel(components=[copy_skycomponent(comp) for comp in sm.components],
images=[copy_image(im) for im in sm.images],
fixed=sm.fixed)
def test_readwriteskymodel_no_image(self):
vis = create_blockvisibility(
self.midcore,
self.times,
self.frequency,
channel_bandwidth=self.channel_bandwidth,
phasecentre=self.phasecentre,
polarisation_frame=PolarisationFrame("linear"),
weight=1.0)
gt = create_gaintable_from_blockvisibility(vis, timeslice='auto')
gt = simulate_gaintable(gt, phase_error=1.0, amplitude_error=0.1)
sm = SkyModel(components=[self.comp], gaintable=gt)
config = {
"buffer": {
"directory": self.dir
},
"skymodel": {
"name": "test_bufferskymodel.hdf",
"data_model": "SkyModel"
}
}
bdm = BufferSkyModel(config["buffer"], config["skymodel"], sm)
bdm.sync()
new_bdm = BufferSkyModel(config["buffer"], config["skymodel"])
new_bdm.sync()
newsm = bdm.memory_data_model
assert newsm.components[0].flux.shape == self.comp.flux.shape
assert newsm.gaintable.data.shape == gt.data.shape
def copy_skymodel(sm):
""" Copy a sky model
"""
if sm.components is not None:
newcomps = [copy_skycomponent(comp) for comp in sm.components]
else:
newcomps = None
if sm.image is not None:
newimage = copy_image(sm.image)
else:
newimage = None
if sm.mask is not None:
newmask = copy_image(sm.mask)
else:
newmask = None
if sm.gaintable is not None:
newgt = copy_gaintable(sm.gaintable)
else:
newgt = None
return SkyModel(components=newcomps, image=newimage, gaintable=newgt, mask=newmask,
fixed=sm.fixed)
def solve_skymodel(vis, skymodel, gain=0.1, **kwargs):
"""Fit a single skymodel to a visibility
:param evis: Expected vis for this ssm
:param calskymodel: scm element being fit i.e. (skymodel, gaintable) tuple
:param gain: Gain in step
:param method: 'fit' or 'sum'
:param kwargs:
:return: skycomponent
"""
if skymodel.fixed:
return skymodel
new_comps = list()
for comp in skymodel.components:
new_comp = copy_skycomponent(comp)
new_comp, _ = fit_visibility(vis, new_comp)
new_comp.flux = gain * new_comp.flux + (1.0 - gain) * comp.flux
new_comps.append(new_comp)
new_images = list()
for im in skymodel.images:
new_image = copy_image(im)
new_image = solve_image(vis, new_image, **kwargs)
new_images.append(new_image)
return SkyModel(components=new_comps, images=new_images)
def test_expand_skymodel_by_skycomponents(self):
beam = create_test_image(cellsize=0.0015,
phasecentre=self.vis.phasecentre,
frequency=self.frequency)
beam = create_low_test_beam(beam, use_local=False)
gleam_components = create_low_test_skycomponents_from_gleam(
flux_limit=1.0,
phasecentre=self.phasecentre,
frequency=self.frequency,
polarisation_frame=PolarisationFrame('stokesI'),
radius=0.2)
pb_gleam_components = apply_beam_to_skycomponent(
gleam_components, beam)
actual_components = filter_skycomponents_by_flux(pb_gleam_components,
flux_min=1.0)
assert len(actual_components) == 37, len(actual_components)
sm = SkyModel(image=self.model, components=actual_components)
assert len(sm.components) == len(actual_components)
scatter_sm = expand_skymodel_by_skycomponents(sm)
assert len(scatter_sm) == len(actual_components) + 1
assert len(scatter_sm[0].components) == 1
def test_readwriteskymodel(self):
im = create_test_image()
sm = SkyModel(components=[self.comp], images=[im, im])
export_skymodel_to_hdf5(sm, '%s/test_skymodel.hdf' % self.dir)
newsm = import_skymodel_from_hdf5('%s/test_skymodel.hdf' % self.dir)
assert newsm.components[0].flux.shape == self.comp.flux.shape
assert newsm.images[0].data.shape == im.data.shape
assert numpy.max(numpy.abs(newsm.images[0].data - im.data)) < 1e-15
def expand_skymodel_by_skycomponents(sm, **kwargs):
""" Expand a sky model so that all components and the image are in separate skymodels
The mask and gaintable are taken to apply for all new skymodels.
:param sm: SkyModel
:return: List of SkyModels
"""
result = [SkyModel(components=[comp],
image=None,
gaintable=copy_gaintable(sm.gaintable),
mask=copy_image(sm.mask),
fixed=sm.fixed) for comp in sm.components]
if sm.image is not None:
result.append(SkyModel(components=None,
image=copy_image(sm.image),
gaintable=copy_gaintable(sm.gaintable),
mask=copy_image(sm.mask),
fixed=sm.fixed))
return result
def test_create(self):
fluxes = numpy.linspace(0, 1.0, 11)
sc = [
create_skycomponent(
direction=self.phasecentre,
flux=numpy.array([[f]]),
frequency=self.frequency,
polarisation_frame=PolarisationFrame('stokesI'))
for f in fluxes
]
sm = SkyModel(image=self.model, components=sc, mask=self.mask)
assert len(sm.components) == 11
def output_images(result):
BufferSkyModel(conf["buffer"], conf['outputs']['skymodel'], SkyModel(images=result[0])).sync()
deconvolved = image_gather_channels(result[0])
residual = image_gather_channels(remove_sumwt(result[1]))
restored = image_gather_channels(result[2])
BufferImage(conf["buffer"], conf['outputs']['deconvolved'], deconvolved).sync()
BufferImage(conf["buffer"], conf['outputs']['residual'], residual).sync()
BufferImage(conf["buffer"], conf['outputs']['restored'], restored).sync()
return result
def test_copy(self):
fluxes = numpy.linspace(0, 1.0, 11)
sc = [create_skycomponent(direction=self.phasecentre, flux=numpy.array([[f]]), frequency=self.frequency,
polarisation_frame=PolarisationFrame('stokesI')) for f in fluxes]
sm = SkyModel(image=self.model, components=sc, mask=self.mask)
sm_copy = copy_skymodel(sm)
assert len(sm.components) == len(sm_copy.components)
sm_fluxes = numpy.array([c.flux[0,0] for c in sm.components])
sm_copy_fluxes = numpy.array([c.flux[0,0] for c in sm_copy.components])
assert numpy.max(numpy.abs(sm_fluxes - sm_copy_fluxes)) < 1e-7
assert numpy.abs(numpy.max(sm.mask.data - 1.0)) < 1e-7
assert numpy.abs(numpy.min(sm.mask.data - 0.0)) < 1e-7
def test_readwriteskymodel(self):
self.vis = create_blockvisibility(self.lowcore, self.times, self.frequency,
channel_bandwidth=self.channel_bandwidth,
phasecentre=self.phasecentre,
polarisation_frame=PolarisationFrame("linear"),
weight=1.0)
im = create_test_image()
gt = create_gaintable_from_blockvisibility(self.vis, timeslice='auto')
sm = SkyModel(components=[self.comp], image=im, gaintable=gt)
export_skymodel_to_hdf5(sm, '%s/test_data_model_helpers_skymodel.hdf' % self.dir)
newsm = import_skymodel_from_hdf5('%s/test_data_model_helpers_skymodel.hdf' % self.dir)
assert newsm.components[0].flux.shape == self.comp.flux.shape
assert newsm.image.data.shape == im.data.shape
assert numpy.max(numpy.abs(newsm.image.data - im.data)) < 1e-15
def import_skymodel_from_hdf5(filename):
"""Import a Skymodel from HDF5 format
:param filename:
:return: SkyModel
"""
with h5py.File(filename, 'r') as f:
ncomponents = f.attrs['number_skycomponents']
components = [convert_hdf_to_skycomponent(f['skycomponent%d' % i])
for i in range(ncomponents)]
nimages = f.attrs['number_images']
images = [convert_hdf_to_image(f['image%d' % i]) for i in range(nimages)]
return SkyModel(components=components, images=images)
def partition_skymodel_by_flux(sc, model, flux_threshold=-numpy.inf):
"""Partition skymodel according to flux
:param sc:
:param model:
:param flux_threshold:
:return:
"""
brightsc = filter_skycomponents_by_flux(sc, flux_min=flux_threshold)
weaksc = filter_skycomponents_by_flux(sc, flux_max=flux_threshold)
log.info('Converted %d components into %d bright components and one image containing %d components'
% (len(sc), len(brightsc), len(weaksc)))
im = copy_image(model)
im = insert_skycomponent(im, weaksc)
return SkyModel(components=[copy_skycomponent(comp) for comp in brightsc],
image=copy_image(im), mask=None,
fixed=False)
def test_expand_skymodel_voronoi(self):
self.model = create_image(
npixel=256,
cellsize=0.001,
polarisation_frame=PolarisationFrame("stokesI"),
frequency=self.frequency,
channel_bandwidth=self.channel_bandwidth,
phasecentre=self.phasecentre)
beam = create_low_test_beam(self.model, use_local=False)
gleam_components = create_low_test_skycomponents_from_gleam(
flux_limit=1.0,
phasecentre=self.phasecentre,
frequency=self.frequency,
polarisation_frame=PolarisationFrame('stokesI'),
radius=0.1)
pb_gleam_components = apply_beam_to_skycomponent(
gleam_components, beam)
actual_components = filter_skycomponents_by_flux(pb_gleam_components,
flux_min=1.0)
_, actual_components = remove_neighbouring_components(
actual_components, 0.05)
for imask, mask in enumerate(
image_voronoi_iter(self.model, actual_components)):
mask.data *= beam.data
assert isinstance(mask, Image)
assert mask.data.dtype == "float"
assert numpy.sum(mask.data) > 1
# import matplotlib.pyplot as plt
# from processing_components.image.operations import show_image
# show_image(mask)
# plt.show()
assert len(actual_components) == 9, len(actual_components)
sm = SkyModel(image=self.model, components=actual_components)
assert len(sm.components) == len(actual_components)
scatter_sm = expand_skymodel_by_skycomponents(sm)
assert len(scatter_sm) == len(actual_components) + 1
assert len(scatter_sm[0].components) == 1
def initialize_skymodel_voronoi(model, comps, gt=None):
"""Create a skymodel by Voronoi partitioning of the components, fill with components
:param model: Model image
:param comps: Skycomponents
:param gt: Gaintable
:return:
"""
skymodel_images = list()
for i, mask in enumerate(image_voronoi_iter(model, comps)):
im = copy_image(model)
im.data *= mask.data
if gt is not None:
newgt = copy_gaintable(gt)
newgt.phasecentre = comps[i].direction
else:
newgt=None
skymodel_images.append(SkyModel(image=im, components=None, gaintable=newgt, mask=mask))
return skymodel_images
def test_readwriteskymodel(self):
im = create_test_image()
sm = SkyModel(components=[self.comp], images=[im, im])
config = {
"buffer": {
"directory": self.dir
},
"skymodel": {
"name": "test_bufferskymodel.hdf",
"data_model": "SkyModel"
}
}
bdm = BufferSkyModel(config["buffer"], config["skymodel"], sm)
bdm.sync()
new_bdm = BufferSkyModel(config["buffer"], config["skymodel"])
new_bdm.sync()
newsm = bdm.memory_data_model
assert newsm.components[0].flux.shape == self.comp.flux.shape
assert newsm.images[0].data.shape == im.data.shape
assert numpy.max(numpy.abs(newsm.images[0].data - im.data)) < 1e-15
def test_invertcal(self):
self.actualSetUp(zerow=True)
skymodel_vislist = predict_skymodel_list_serial_workflow(
self.vis_list[0], self.skymodel_list, context='2d', docal=True)
result_skymodel = [
SkyModel(components=None, image=self.skymodel_list[-1].image)
for v in skymodel_vislist
]
results = invert_skymodel_list_serial_workflow(skymodel_vislist,
result_skymodel,
context='2d',
docal=True)
assert numpy.max(numpy.abs(results[0][0].data)) > 0.0
assert numpy.max(numpy.abs(results[0][1])) > 0.0
if self.plot:
import matplotlib.pyplot as plt
from wrappers.serial.image.operations import show_image
show_image(results[0][0],
title='Dirty image, no cross-subtraction',
vmax=0.1,
vmin=-0.01)
plt.show()
def create_low_test_skymodel_from_gleam(npixel=512,
polarisation_frame=PolarisationFrame(
"stokesI"),
cellsize=0.000015,
frequency=numpy.array([1e8]),
channel_bandwidth=numpy.array([1e6]),
phasecentre=None,
kind='cubic',
applybeam=True,
flux_limit=0.1,
flux_max=numpy.inf,
flux_threshold=1.0,
insert_method='Nearest',
telescope='LOW') -> SkyModel:
"""Create LOW test skymodel from the GLEAM survey
Stokes I is estimated from a cubic spline fit to the measured fluxes. The polarised flux is always zero.
See http://www.mwatelescope.org/science/gleam-survey The catalog is available from Vizier.
VIII/100 GaLactic and Extragalactic All-sky MWA survey (Hurley-Walker+, 2016)
GaLactic and Extragalactic All-sky Murchison Wide Field Array (GLEAM) survey. I: A low-frequency extragalactic
catalogue. Hurley-Walker N., et al., Mon. Not. R. Astron. Soc., 464, 1146-1167 (2017), 2017MNRAS.464.1146H
:param telescope:
:param npixel: Number of pixels
:param polarisation_frame: Polarisation frame (default PolarisationFrame("stokesI"))
:param cellsize: cellsize in radians
:param frequency:
:param channel_bandwidth: Channel width (Hz)
:param phasecentre: phasecentre (SkyCoord)
:param kind: Kind of interpolation (see scipy.interpolate.interp1d) Default: cubic
:param applybeam: Apply the primary beam?
:param flux_limit: Weakest component
:param flux_max: Maximum strength component to be included in components
:param flux_threshold: Split between components (brighter) and image (weaker)
:param insert_method: Nearest | PSWF | Lanczos
:return:
:return: SkyModel
"""
if phasecentre is None:
phasecentre = SkyCoord(ra=+15.0 * u.deg,
dec=-35.0 * u.deg,
frame='icrs',
equinox='J2000')
radius = npixel * cellsize
sc = create_low_test_skycomponents_from_gleam(
flux_limit=flux_limit,
polarisation_frame=polarisation_frame,
frequency=frequency,
phasecentre=phasecentre,
kind=kind,
radius=radius)
sc = filter_skycomponents_by_flux(sc, flux_max=flux_max)
if polarisation_frame is None:
polarisation_frame = PolarisationFrame("stokesI")
npol = polarisation_frame.npol
nchan = len(frequency)
shape = [nchan, npol, npixel, npixel]
w = WCS(naxis=4)
# The negation in the longitude is needed by definition of RA, DEC
w.wcs.cdelt = [
-cellsize * 180.0 / numpy.pi, cellsize * 180.0 / numpy.pi, 1.0,
channel_bandwidth[0]
]
w.wcs.crpix = [npixel // 2 + 1, npixel // 2 + 1, 1.0, 1.0]
w.wcs.ctype = ["RA---SIN", "DEC--SIN", 'STOKES', 'FREQ']
w.wcs.crval = [phasecentre.ra.deg, phasecentre.dec.deg, 1.0, frequency[0]]
w.naxis = 4
w.wcs.radesys = 'ICRS'
w.wcs.equinox = 2000.0
model = create_image_from_array(numpy.zeros(shape),
w,
polarisation_frame=polarisation_frame)
if applybeam:
beam = create_pb(model, telescope=telescope)
sc = apply_beam_to_skycomponent(sc, beam)
weaksc = filter_skycomponents_by_flux(sc, flux_max=flux_threshold)
brightsc = filter_skycomponents_by_flux(sc, flux_min=flux_threshold)
model = insert_skycomponent(model, weaksc, insert_method=insert_method)
log.info(
'create_low_test_skymodel_from_gleam: %d bright sources above flux threshold %.3f, %d weak sources below '
% (len(brightsc), flux_threshold, len(weaksc)))
return SkyModel(components=brightsc,
image=model,
mask=None,
gaintable=None)
def actualSetup(self, nsources=None, nvoronoi=None):
n_workers = 8
# Set up the observation: 10 minutes at transit, with 10s integration.
# Skip 5/6 points to avoid outstation redundancy
nfreqwin = 1
ntimes = 3
self.rmax = 2500.0
dec = -40.0 * u.deg
frequency = [1e8]
channel_bandwidth = [0.1e8]
times = numpy.linspace(-10.0, 10.0,
ntimes) * numpy.pi / (3600.0 * 12.0)
phasecentre = SkyCoord(ra=+0.0 * u.deg,
dec=dec,
frame='icrs',
equinox='J2000')
low = create_named_configuration('LOWBD2', rmax=self.rmax)
centre = numpy.mean(low.xyz, axis=0)
distance = numpy.hypot(low.xyz[:, 0] - centre[0],
low.xyz[:, 1] - centre[1],
low.xyz[:, 2] - centre[2])
lowouter = low.data[distance > 1000.0][::6]
lowcore = low.data[distance < 1000.0][::3]
low.data = numpy.hstack((lowcore, lowouter))
blockvis = create_blockvisibility(
low,
times,
frequency=frequency,
channel_bandwidth=channel_bandwidth,
weight=1.0,
phasecentre=phasecentre,
polarisation_frame=PolarisationFrame("stokesI"),
zerow=True)
vis = convert_blockvisibility_to_visibility(blockvis)
advice = advise_wide_field(vis, guard_band_image=2.0, delA=0.02)
cellsize = advice['cellsize']
npixel = advice['npixels2']
small_model = create_image_from_visibility(blockvis,
npixel=512,
frequency=frequency,
nchan=nfreqwin,
cellsize=cellsize,
phasecentre=phasecentre)
vis.data['imaging_weight'][...] = vis.data['weight'][...]
vis = weight_list_serial_workflow([vis], [small_model])[0]
vis = taper_list_serial_workflow([vis], 3 * cellsize)[0]
blockvis = convert_visibility_to_blockvisibility(vis)
# ### Generate the model from the GLEAM catalog, including application of the primary beam.
beam = create_image_from_visibility(blockvis,
npixel=npixel,
frequency=frequency,
nchan=nfreqwin,
cellsize=cellsize,
phasecentre=phasecentre)
beam = create_low_test_beam(beam, use_local=False)
flux_limit = 0.5
original_gleam_components = create_low_test_skycomponents_from_gleam(
flux_limit=flux_limit,
phasecentre=phasecentre,
frequency=frequency,
polarisation_frame=PolarisationFrame('stokesI'),
radius=0.15)
all_components = apply_beam_to_skycomponent(original_gleam_components,
beam)
all_components = filter_skycomponents_by_flux(all_components,
flux_min=flux_limit)
voronoi_components = filter_skycomponents_by_flux(all_components,
flux_min=1.5)
def max_flux(elem):
return numpy.max(elem.flux)
voronoi_components = sorted(voronoi_components,
key=max_flux,
reverse=True)
if nsources is not None:
all_components = [all_components[0]]
if nvoronoi is not None:
voronoi_components = [voronoi_components[0]]
self.screen = import_image_from_fits(
arl_path('data/models/test_mpc_screen.fits'))
all_gaintables = create_gaintable_from_screen(blockvis, all_components,
self.screen)
gleam_skymodel_noniso = [
SkyModel(components=[all_components[i]],
gaintable=all_gaintables[i])
for i, sm in enumerate(all_components)
]
# ### Now predict the visibility for each skymodel and apply the gaintable for that skymodel,
# returning a list of visibilities, one for each skymodel. We then sum these to obtain
# the total predicted visibility. All images and skycomponents in the same skymodel
# get the same gaintable applied which means that in this case each skycomponent has a separate gaintable.
self.all_skymodel_noniso_vis = convert_blockvisibility_to_visibility(
blockvis)
ngroup = n_workers
future_vis = arlexecute.scatter(self.all_skymodel_noniso_vis)
chunks = [
gleam_skymodel_noniso[i:i + ngroup]
for i in range(0, len(gleam_skymodel_noniso), ngroup)
]
for chunk in chunks:
result = predict_skymodel_list_arlexecute_workflow(future_vis,
chunk,
context='2d',
docal=True)
work_vis = arlexecute.compute(result, sync=True)
for w in work_vis:
self.all_skymodel_noniso_vis.data['vis'] += w.data['vis']
assert numpy.max(
numpy.abs(self.all_skymodel_noniso_vis.data['vis'])) > 0.0
self.all_skymodel_noniso_blockvis = convert_visibility_to_blockvisibility(
self.all_skymodel_noniso_vis)
# ### Remove weaker of components that are too close (0.02 rad)
idx, voronoi_components = remove_neighbouring_components(
voronoi_components, 0.02)
model = create_image_from_visibility(blockvis,
npixel=npixel,
frequency=frequency,
nchan=nfreqwin,
cellsize=cellsize,
phasecentre=phasecentre)
# Use the gaintable for the brightest component as the starting gaintable
all_gaintables[0].gain[...] = numpy.conjugate(
all_gaintables[0].gain[...])
all_gaintables[0].gain[...] = 1.0 + 0.0j
self.theta_list = initialize_skymodel_voronoi(model,
voronoi_components,
all_gaintables[0])
def actualSetup(self,
vnchan=1,
doiso=True,
ntimes=5,
flux_limit=2.0,
zerow=True,
fixed=False):
nfreqwin = vnchan
rmax = 300.0
npixel = 512
cellsize = 0.001
frequency = numpy.linspace(0.8e8, 1.2e8, nfreqwin)
if nfreqwin > 1:
channel_bandwidth = numpy.array(nfreqwin *
[frequency[1] - frequency[0]])
else:
channel_bandwidth = [0.4e8]
times = numpy.linspace(-numpy.pi / 3.0, numpy.pi / 3.0, ntimes)
phasecentre = SkyCoord(ra=-60.0 * u.deg,
dec=-60.0 * u.deg,
frame='icrs',
equinox='J2000')
lowcore = create_named_configuration('LOWBD2', rmax=rmax)
block_vis = create_blockvisibility(
lowcore,
times,
frequency=frequency,
channel_bandwidth=channel_bandwidth,
weight=1.0,
phasecentre=phasecentre,
polarisation_frame=PolarisationFrame("stokesI"),
zerow=zerow)
block_vis.data['uvw'][..., 2] = 0.0
self.beam = create_image_from_visibility(
block_vis,
npixel=npixel,
frequency=[numpy.average(frequency)],
nchan=nfreqwin,
channel_bandwidth=[numpy.sum(channel_bandwidth)],
cellsize=cellsize,
phasecentre=phasecentre)
self.components = create_low_test_skycomponents_from_gleam(
flux_limit=flux_limit,
phasecentre=phasecentre,
frequency=frequency,
polarisation_frame=PolarisationFrame('stokesI'),
radius=npixel * cellsize)
self.beam = create_low_test_beam(self.beam)
self.components = apply_beam_to_skycomponent(self.components,
self.beam,
flux_limit=flux_limit)
self.vis = copy_visibility(block_vis, zero=True)
gt = create_gaintable_from_blockvisibility(block_vis, timeslice='auto')
for i, sc in enumerate(self.components):
if sc.flux[0, 0] > 10:
sc.flux[...] /= 10.0
component_vis = copy_visibility(block_vis, zero=True)
gt = simulate_gaintable(gt,
amplitude_error=0.0,
phase_error=0.1,
seed=None)
component_vis = predict_skycomponent_visibility(component_vis, sc)
component_vis = apply_gaintable(component_vis, gt)
self.vis.data['vis'][...] += component_vis.data['vis'][...]
# Do an isoplanatic selfcal
self.model_vis = copy_visibility(self.vis, zero=True)
self.model_vis = predict_skycomponent_visibility(
self.model_vis, self.components)
if doiso:
gt = solve_gaintable(self.vis,
self.model_vis,
phase_only=True,
timeslice='auto')
self.vis = apply_gaintable(self.vis, gt, inverse=True)
self.model_vis = convert_blockvisibility_to_visibility(self.model_vis)
self.model_vis, _, _ = weight_visibility(self.model_vis, self.beam)
self.dirty_model, sumwt = invert_function(self.model_vis,
self.beam,
context='2d')
export_image_to_fits(self.dirty_model,
"%s/test_skymodel-model_dirty.fits" % self.dir)
lvis = convert_blockvisibility_to_visibility(self.vis)
lvis, _, _ = weight_visibility(lvis, self.beam)
dirty, sumwt = invert_function(lvis, self.beam, context='2d')
if doiso:
export_image_to_fits(
dirty, "%s/test_skymodel-initial-iso-residual.fits" % self.dir)
else:
export_image_to_fits(
dirty,
"%s/test_skymodel-initial-noiso-residual.fits" % self.dir)
self.skymodels = [
SkyModel(components=[cm], fixed=fixed) for cm in self.components
]
gleam_model = [arlexecute.execute(create_low_test_image_from_gleam)(npixel=npixel,
frequency=[frequency[f]],
channel_bandwidth=[channel_bandwidth[f]],
cellsize=cellsize,
phasecentre=phasecentre,
polarisation_frame=PolarisationFrame("stokesI"),
flux_limit=1.0,
applybeam=True)
for f, freq in enumerate(frequency)]
print('About to make GLEAM model')
gleam_model = arlexecute.persist(gleam_model)
print('About to run predict to get predicted visibility')
vis_list = arlexecute.scatter(vis_list)
predicted_vislist = predict_list_arlexecute_workflow(vis_list, gleam_model, context='wstack',
vis_slices=vis_slices)
predicted_vislist = arlexecute.persist(predicted_vislist)
corrupted_vislist = corrupt_list_arlexecute_workflow(predicted_vislist, phase_error=1.0)
print('About to run corrupt to get corrupted visibility')
corrupted_vislist = arlexecute.persist(corrupted_vislist)
corrupted_vislist = arlexecute.compute(corrupted_vislist, sync=True)
export_blockvisibility_to_hdf5(corrupted_vislist, 'gleam_simulation_vislist.hdf')
gleam_model = arlexecute.compute(gleam_model, sync=True)
gleam_skymodel = SkyModel(image=gleam_model)
export_skymodel_to_hdf5(gleam_skymodel, 'gleam_simulation_skymodel.hdf')
arlexecute.close()
channel_bandwidth=[channel_bandwidth[f]],
cellsize=cellsize,
phasecentre=phasecentre,
polarisation_frame=PolarisationFrame("stokesI"),
flux_limit=3.0,
applybeam=True) for f, freq in enumerate(frequency)
]
# Put the model on the cluster
dprepb_model = arlexecute.persist(dprepb_model)
print('About to make GLEAM model')
zero_model = [
arlexecute.execute(create_empty_image_like)(im) for im in dprepb_model
]
zero_model = arlexecute.compute(zero_model, sync=True)
zero_skymodel = SkyModel(images=zero_model)
export_skymodel_to_hdf5(
zero_skymodel,
arl_path('%s/ska-pipeline_simulation_skymodel.hdf' % results_dir))
# In[ ]:
# vis_list = arlexecute.scatter(vis_list)
wstack = True
if wstack:
print('Using w stack with %d slices' % vis_slices)
predicted_vislist = predict_list_arlexecute_workflow(
vis_list, dprepb_model, context='wstack', vis_slices=vis_slices)
else:
print('Using timeslicing with %d slices' % ntimes)
all_components = apply_beam_to_skycomponent(original_gleam_components,
beam)
all_components = filter_skycomponents_by_flux(all_components,
flux_min=flux_limit)
all_components = sorted(all_components,
key=lambda comp: numpy.max(comp.flux),
reverse=True)
print("Number of components in simulation %d" % len(all_components))
screen = import_image_from_fits(
arl_path('data/models/test_mpc_screen.fits'))
all_gaintables = create_gaintable_from_screen(block_vis, all_components,
screen)
all_skymodel = [
SkyModel(components=[all_components[i]], gaintable=all_gaintables[i])
for i, sm in enumerate(all_components)
]
#######################################################################################################
# Calculate visibility by using the predict_skymodel function which applies a different gaintable table
# for each skymodel. We do the calculation in chunks of nworkers skymodels.
all_skymodel_blockvis = copy_visibility(block_vis, zero=True)
all_skymodel_vis = convert_blockvisibility_to_visibility(
all_skymodel_blockvis)
ngroup = 8
future_vis = arlexecute.scatter(all_skymodel_vis)
chunks = [
all_skymodel[i:i + ngroup] for i in range(0, len(all_skymodel), ngroup)
]