def predict_vislist_wrapper(conf):
"""Wrapper for prediction
:param conf: Configuration from JSON file
:return:
"""
vis_list = buffer_data_model_to_memory(conf["buffer"],
conf['inputs']['vis_list'])
skymodel = buffer_data_model_to_memory(conf["buffer"],
conf['inputs']['skymodel'])
flux_limit = conf['primary_beam']['flux_limit']
if conf["primary_beam"]["apply"]:
def apply_pb_image(vt, model):
telescope = vt.configuration.name
pb = create_pb(model, telescope)
model.data *= pb.data
return model
def apply_pb_comp(vt, model, comp):
telescope = vt.configuration.name
pb = create_pb(model, telescope)
return apply_beam_to_skycomponent(comp, pb, flux_limit)
image_list = [
arlexecute.execute(apply_pb_image, nout=1)(v, skymodel.images[i])
for i, v in enumerate(vis_list)
]
if len(skymodel.components) > 1:
component_list = [
arlexecute.execute(apply_pb_comp,
nout=1)(v, skymodel.images[i],
skymodel.components)
for i, v in enumerate(vis_list)
]
else:
component_list = []
else:
image_list = [skymodel.images[0] for v in vis_list]
component_list = skymodel.components
future_vis_list = arlexecute.scatter(vis_list)
predicted_vis_list = [
arlexecute.execute(predict_skycomponent_visibility)(v, component_list)
for v in future_vis_list
]
predicted_vis_list = predict_component(
predicted_vis_list,
image_list,
context=conf['imaging']['context'],
vis_slices=conf['imaging']['vis_slices'])
return arlexecute.execute(memory_data_model_to_buffer)(
predicted_vis_list, conf["buffer"], conf["outputs"]["vis_list"])
def continuum_imaging_wrapper(conf):
"""Wrap continuum imaging pipeline
:param conf:
:return:
"""
vis_list = buffer_data_model_to_memory(conf["buffer"],
conf['inputs']['vis_list'])
cellsize = json_to_quantity(conf["image"]["cellsize"]).to("rad").value
npixel = conf["image"]["npixel"]
pol_frame = PolarisationFrame(conf["image"]["polarisation_frame"])
model_imagelist = [
arlexecute.execute(create_image_from_visibility)(
v, npixel=npixel, cellsize=cellsize, polarisation_frame=pol_frame)
for v in vis_list
]
future_vis_list = arlexecute.scatter(vis_list)
result = continuum_imaging_component(
vis_list=future_vis_list,
model_imagelist=model_imagelist,
context=conf["imaging"]["context"],
scales=conf["deconvolution"]["scales"],
algorithm=conf["deconvolution"]["algorithm"],
nmoment=conf["deconvolution"]["nmoment"],
niter=conf["deconvolution"]["niter"],
fractional_threshold=conf["deconvolution"]["fractional_threshold"],
threshold=conf["deconvolution"]["threshold"],
nmajor=conf["deconvolution"]["nmajor"],
gain=conf["deconvolution"]["gain"],
deconvolve_facets=conf["deconvolution"]["deconvolve_facets"],
deconvolve_overlap=conf["deconvolution"]["deconvolve_overlap"],
deconvolve_taper=conf["deconvolution"]["deconvolve_taper"],
vis_slices=conf["imaging"]["vis_slices"],
psf_support=conf["deconvolution"]["psf_support"])
def output_images(result):
deconvolved = image_gather_channels(result[0])
residual = image_gather_channels(remove_sumwt(result[1]))
restored = image_gather_channels(result[2])
memory_data_model_to_buffer(deconvolved, conf["buffer"],
conf['outputs']['deconvolved'])
memory_data_model_to_buffer(restored, conf["buffer"],
conf['outputs']['restored'])
memory_data_model_to_buffer(residual, conf["buffer"],
conf['outputs']['residual'])
return result
return arlexecute.execute(output_images)(result)
def create_vislist_wrapper(conf):
""" Create an empty vislist
:param conf: Configuration from JSON file
:return:
"""
configuration = conf['create_vislist']['configuration']
rmax = conf['create_vislist']['rmax']
phasecentre = json_to_skycoord(conf['create_vislist']['phasecentre'])
frequency = json_to_linspace(conf['create_vislist']['frequency'])
if conf['create_vislist']['frequency']['steps'] > 1:
channel_bandwidth = numpy.array(
conf['create_vislist']['frequency']['steps'] *
[frequency[1] - frequency[0]])
else:
channel_bandwidth = numpy.array(
conf['create_vislist']['frequency']['start'])
times = json_to_linspace(conf['create_vislist']['times'])
vis_list = simulate_component(configuration,
rmax=rmax,
frequency=frequency,
channel_bandwidth=channel_bandwidth,
times=times,
phasecentre=phasecentre,
order='frequency')
return arlexecute.execute(memory_data_model_to_buffer)(
vis_list, conf["buffer"], conf["outputs"]["vis_list"])
def test_useFunction(self):
def square(x):
return x**2
arlexecute.set_client(use_dask=False)
graph = arlexecute.execute(square)(numpy.arange(10))
assert (arlexecute.compute(graph) == numpy.array(
[0, 1, 4, 9, 16, 25, 36, 49, 64, 81])).all()
arlexecute.close()
def test_useDaskSync(self):
def square(x):
return x**2
arlexecute.set_client(use_dask=True)
graph = arlexecute.execute(square)(numpy.arange(10))
result = arlexecute.compute(graph, sync=True)
assert (result == numpy.array([0, 1, 4, 9, 16, 25, 36, 49, 64,
81])).all()
arlexecute.close()
def corrupt_vislist_wrapper(conf):
"""Wrapper for corruption
:param conf:
:return:
"""
vis_list = buffer_data_model_to_memory(conf["buffer"],
conf['inputs']['vis_list'])
phase_error = json_to_quantity(
conf['corrupt_vislist']['phase_error']).to('rad').value
corrupted_vislist = corrupt_component(
vis_list,
phase_error=phase_error,
amplitude_error=conf['corrupt_vislist']['amplitude_error'])
return arlexecute.execute(memory_data_model_to_buffer)(
corrupted_vislist, conf["buffer"], conf['outputs']['vis_list'])
def setUp(self):
arlexecute.set_client(use_dask=False)
from data_models.parameters import arl_path
self.dir = arl_path('test_results')
self.lowcore = create_named_configuration('LOWBD2-CORE')
self.times = numpy.linspace(-3, +3, 13) * (numpy.pi / 12.0)
self.frequency = numpy.array([1e8])
self.channel_bandwidth = numpy.array([1e7])
# Define the component and give it some polarisation and spectral behaviour
f = numpy.array([100.0])
self.flux = numpy.array([f])
self.phasecentre = SkyCoord(ra=+15.0 * u.deg,
dec=-35.0 * u.deg,
frame='icrs',
equinox='J2000')
self.compabsdirection = SkyCoord(ra=17.0 * u.deg,
dec=-36.5 * u.deg,
frame='icrs',
equinox='J2000')
self.comp = create_skycomponent(
direction=self.compabsdirection,
flux=self.flux,
frequency=self.frequency,
polarisation_frame=PolarisationFrame('stokesI'))
self.image = create_test_image(
frequency=self.frequency,
phasecentre=self.phasecentre,
cellsize=0.001,
polarisation_frame=PolarisationFrame('stokesI'))
self.image.data[self.image.data < 0.0] = 0.0
self.image_graph = arlexecute.execute(create_test_image)(
frequency=self.frequency,
phasecentre=self.phasecentre,
cellsize=0.001,
polarisation_frame=PolarisationFrame('stokesI'))
def test_calskymodel_solve_component(self):
self.actualSetup(doiso=True)
self.skymodel_list = [
arlexecute.execute(SkyModel, nout=1)(components=[cm])
for cm in self.components
]
calskymodel_list = calskymodel_solve_component(
self.vis, skymodel_list=self.skymodel_list, niter=30, gain=0.25)
skymodel, residual_vis = arlexecute.compute(calskymodel_list,
sync=True)
residual_vis = convert_blockvisibility_to_visibility(residual_vis)
residual_vis, _, _ = weight_visibility(residual_vis, self.beam)
dirty, sumwt = invert_function(residual_vis, self.beam, context='2d')
export_image_to_fits(
dirty, "%s/test_calskymodel-%s-final-iso-residual.fits" %
(self.dir, arlexecute.type()))
qa = qa_image(dirty)
assert qa.data['rms'] < 3.2e-3, qa
arlexecute.set_client(get_dask_Client())
arlexecute.run(init_logging)
import pickle
# Load data from previous simulation
vislist = import_blockvisibility_from_hdf5('gleam_simulation_vislist.hdf')
print(vislist[0])
cellsize = 0.001
npixel = 1024
pol_frame = PolarisationFrame("stokesI")
model_list = [
arlexecute.execute(create_image_from_visibility)(
v, npixel=1024, cellsize=cellsize, polarisation_frame=pol_frame)
for v in vislist
]
controls = create_calibration_controls()
controls['T']['first_selfcal'] = 1
controls['G']['first_selfcal'] = 3
controls['B']['first_selfcal'] = 4
controls['T']['timescale'] = 'auto'
controls['G']['timescale'] = 'auto'
controls['B']['timescale'] = 1e5
pp.pprint(controls)
def actualSetUp(self,
add_errors=False,
freqwin=1,
block=False,
dospectral=True,
dopol=False,
zerow=False):
arlexecute.set_client(use_dask=True)
self.npixel = 256
self.low = create_named_configuration('LOWBD2', rmax=750.0)
self.freqwin = freqwin
self.vis_list = list()
self.ntimes = 5
self.times = numpy.linspace(-3.0, +3.0, self.ntimes) * numpy.pi / 12.0
if freqwin > 1:
self.frequency = numpy.linspace(0.8e8, 1.2e8, self.freqwin)
self.channelwidth = numpy.array(
freqwin * [self.frequency[1] - self.frequency[0]])
else:
self.frequency = numpy.array([0.8e8])
self.channelwidth = numpy.array([1e6])
if dopol:
self.vis_pol = PolarisationFrame('linear')
self.image_pol = PolarisationFrame('stokesIQUV')
f = numpy.array([100.0, 20.0, -10.0, 1.0])
else:
self.vis_pol = PolarisationFrame('stokesI')
self.image_pol = PolarisationFrame('stokesI')
f = numpy.array([100.0])
if dospectral:
flux = numpy.array(
[f * numpy.power(freq / 1e8, -0.7) for freq in self.frequency])
else:
flux = numpy.array([f])
self.phasecentre = SkyCoord(ra=+180.0 * u.deg,
dec=-60.0 * u.deg,
frame='icrs',
equinox='J2000')
self.vis_list = [
arlexecute.execute(ingest_unittest_visibility)(
self.low, [self.frequency[freqwin]],
[self.channelwidth[freqwin]],
self.times,
self.vis_pol,
self.phasecentre,
block=block,
zerow=zerow) for freqwin, _ in enumerate(self.frequency)
]
self.model_graph = [
arlexecute.execute(create_unittest_model,
nout=freqwin)(self.vis_list[freqwin],
self.image_pol,
npixel=self.npixel)
for freqwin, _ in enumerate(self.frequency)
]
self.components_graph = [
arlexecute.execute(create_unittest_components)(
self.model_graph[freqwin], flux[freqwin, :][numpy.newaxis, :])
for freqwin, _ in enumerate(self.frequency)
]
self.model_graph = [
arlexecute.execute(insert_skycomponent,
nout=1)(self.model_graph[freqwin],
self.components_graph[freqwin])
for freqwin, _ in enumerate(self.frequency)
]
self.vis_list = [
arlexecute.execute(predict_skycomponent_visibility)(
self.vis_list[freqwin], self.components_graph[freqwin])
for freqwin, _ in enumerate(self.frequency)
]
# Calculate the model convolved with a Gaussian.
self.model = arlexecute.compute(self.model_graph[0], sync=True)
self.cmodel = smooth_image(self.model)
export_image_to_fits(self.model,
'%s/test_imaging_model.fits' % self.dir)
export_image_to_fits(self.cmodel,
'%s/test_imaging_cmodel.fits' % self.dir)
if add_errors and block:
self.vis_list = [
arlexecute.execute(insert_unittest_errors)(self.vis_list[i])
for i, _ in enumerate(self.frequency)
]
self.vis = arlexecute.compute(self.vis_list[0], sync=True)
self.components = arlexecute.compute(self.components_graph[0],
sync=True)
def actualSetUp(self,
add_errors=False,
freqwin=5,
block=False,
dospectral=True,
dopol=False,
amp_errors=None,
phase_errors=None,
zerow=True):
if amp_errors is None:
amp_errors = {'T': 0.0, 'G': 0.01, 'B': 0.01}
if phase_errors is None:
phase_errors = {'T': 1.0, 'G': 0.1, 'B': 0.01}
self.npixel = 512
self.low = create_named_configuration('LOWBD2', rmax=750.0)
self.freqwin = freqwin
self.vis_list = list()
self.ntimes = 5
self.times = numpy.linspace(-3.0, +3.0, self.ntimes) * numpy.pi / 12.0
self.frequency = numpy.linspace(0.8e8, 1.2e8, self.freqwin)
if freqwin > 1:
self.channelwidth = numpy.array(
freqwin * [self.frequency[1] - self.frequency[0]])
else:
self.channelwidth = numpy.array([1e6])
if dopol:
self.vis_pol = PolarisationFrame('linear')
self.image_pol = PolarisationFrame('stokesIQUV')
f = numpy.array([100.0, 20.0, -10.0, 1.0])
else:
self.vis_pol = PolarisationFrame('stokesI')
self.image_pol = PolarisationFrame('stokesI')
f = numpy.array([100.0])
if dospectral:
flux = numpy.array(
[f * numpy.power(freq / 1e8, -0.7) for freq in self.frequency])
else:
flux = numpy.array([f])
self.phasecentre = SkyCoord(ra=+180.0 * u.deg,
dec=-60.0 * u.deg,
frame='icrs',
equinox='J2000')
self.vis_list = [
arlexecute.execute(ingest_unittest_visibility)(
self.low, [self.frequency[i]], [self.channelwidth[i]],
self.times,
self.vis_pol,
self.phasecentre,
block=block,
zerow=zerow) for i, _ in enumerate(self.frequency)
]
self.model_imagelist = [
arlexecute.execute(create_unittest_model,
nout=freqwin)(self.vis_list[0],
self.image_pol,
npixel=self.npixel)
for i, _ in enumerate(self.frequency)
]
self.components_list = [
arlexecute.execute(create_unittest_components)(
self.model_imagelist[i], flux[i, :][numpy.newaxis, :])
for i, _ in enumerate(self.frequency)
]
# Apply the LOW primary beam and insert into model
self.model_imagelist = [
arlexecute.execute(insert_skycomponent,
nout=1)(self.model_imagelist[freqwin],
self.components_list[freqwin])
for freqwin, _ in enumerate(self.frequency)
]
self.vis_list = [
arlexecute.execute(predict_skycomponent_visibility)(
self.vis_list[freqwin], self.components_list[freqwin])
for freqwin, _ in enumerate(self.frequency)
]
# Calculate the model convolved with a Gaussian.
self.model_imagelist = arlexecute.compute(self.model_imagelist,
sync=True)
model = self.model_imagelist[0]
self.cmodel = smooth_image(model)
export_image_to_fits(model,
'%s/test_imaging_delayed_model.fits' % self.dir)
export_image_to_fits(self.cmodel,
'%s/test_imaging_delayed_cmodel.fits' % self.dir)
if add_errors and block:
self.vis_list = [
arlexecute.execute(insert_unittest_errors)(
self.vis_list[i],
amp_errors=amp_errors,
phase_errors=phase_errors)
for i, _ in enumerate(self.frequency)
]
self.vis_list = arlexecute.compute(self.vis_list, sync=True)
self.vis_list = arlexecute.scatter(self.vis_list)
self.model_imagelist = arlexecute.scatter(self.model_imagelist)
def create_skymodel_wrapper(conf):
""" Wrapper to create skymodel
:param conf: Configuration from JSON file
:return:
"""
cellsize = json_to_quantity(conf["image"]["cellsize"]).to("rad").value
npixel = conf["image"]["npixel"]
pol_frame = PolarisationFrame(conf["image"]["polarisation_frame"])
phasecentre = json_to_skycoord(conf['image']['phasecentre'])
frequency = json_to_linspace(conf['image']['frequency'])
if conf['image']['frequency']['steps'] > 1:
channel_bandwidth = numpy.array(conf['image']['frequency']['steps'] *
[frequency[1] - frequency[0]])
else:
channel_bandwidth = numpy.array(conf['image']['frequency']['start'])
flux_limit = json_to_quantity(
conf['create_skymodel']['flux_limit']).to("Jy").value
radius = json_to_quantity(
conf['create_skymodel']['radius']).to('rad').value
kind = conf['create_skymodel']['kind']
models = [
arlexecute.execute(create_image)(
npixel=npixel,
frequency=[frequency[f]],
channel_bandwidth=[channel_bandwidth[f]],
cellsize=cellsize,
phasecentre=phasecentre,
polarisation_frame=pol_frame) for f, freq in enumerate(frequency)
]
catalog = conf['create_skymodel']["catalog"]
if catalog == "gleam":
components = arlexecute.execute(
create_low_test_skycomponents_from_gleam)(
phasecentre=phasecentre,
polarisation_frame=pol_frame,
flux_limit=flux_limit,
frequency=frequency,
kind=kind,
radius=radius)
if conf['create_skymodel']["fill_image"]:
models = [
arlexecute.execute(insert_skycomponent)(m, components)
for m in models
]
elif catalog == "empty":
components = []
else:
raise RuntimeError("Catalog %s is not supported" % catalog)
def output_skymodel(model_list, comp_list):
if conf['create_skymodel']["fill_image"]:
skymodel = SkyModel(images=model_list, components=[])
else:
skymodel = SkyModel(images=[], components=comp_list)
return memory_data_model_to_buffer(skymodel, conf["buffer"],
conf["outputs"]["skymodel"])
return arlexecute.execute(output_skymodel)(models, components)
advice_high = advise_wide_field(vis_list[-1], guard_band_image=8.0, delA=0.02,
wprojection_planes=wprojection_planes)
vis_slices = advice_low['vis_slices']
npixel = advice_high['npixels2']
cellsize = min(advice_low['cellsize'], advice_high['cellsize'])
# Now make a graph to fill with a model drawn from GLEAM
# In[ ]:
gleam_model = [arlexecute.execute(create_low_test_image_from_gleam)(npixel=npixel,
frequency=[frequency[f]],
channel_bandwidth=[channel_bandwidth[f]],
cellsize=cellsize,
phasecentre=phasecentre,
polarisation_frame=PolarisationFrame("stokesI"),
flux_limit=1.0,
applybeam=True)
for f, freq in enumerate(frequency)]
log.info('About to make GLEAM model')
gleam_model = arlexecute.compute(gleam_model, sync=True)
gleam_skymodel = SkyModel(images=gleam_model)
export_skymodel_to_hdf5(gleam_skymodel, 'gleam_simulation_skymodel.hdf')
future_gleam_model = arlexecute.scatter(gleam_model)
# In[ ]:
log.info('About to run predict to get predicted visibility')
future_vis_graph = arlexecute.scatter(vis_list)
predicted_vislist = predict_component(future_vis_graph, gleam_model, context='wstack', vis_slices=vis_slices)